From c0b32213e48d06114d9e6435ef59d7b0b72256b5 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 10 Jan 2018 14:40:34 +0100 Subject: [PATCH 01/28] Updated all refs to the IPDB Transaction Spec & its URL --- docs/root/source/assets.rst | 2 +- docs/root/source/permissions.rst | 4 ++-- docs/root/source/smart-contracts.rst | 2 +- docs/root/source/transaction-concepts.md | 6 +++--- docs/server/source/appendices/cryptography.rst | 5 ++--- docs/server/source/appendices/json-serialization.rst | 5 ++--- docs/server/source/data-models/asset-model.rst | 3 +-- docs/server/source/data-models/block-model.rst | 12 ++++++------ docs/server/source/data-models/conditions.rst | 3 +-- docs/server/source/data-models/inputs-outputs.rst | 3 +-- docs/server/source/data-models/transaction-model.rst | 6 +++--- docs/server/source/data-models/vote-model.rst | 9 ++++----- 12 files changed, 27 insertions(+), 33 deletions(-) diff --git a/docs/root/source/assets.rst b/docs/root/source/assets.rst index a0aa5a31..dd1a241d 100644 --- a/docs/root/source/assets.rst +++ b/docs/root/source/assets.rst @@ -13,4 +13,4 @@ BigchainDB can store data of any kind (within reason), but it's designed to be p .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the note about "owners" in the `IPDB Transaction Spec `_. diff --git a/docs/root/source/permissions.rst b/docs/root/source/permissions.rst index 02029e10..68d47bc8 100644 --- a/docs/root/source/permissions.rst +++ b/docs/root/source/permissions.rst @@ -15,8 +15,8 @@ To spend/transfer an unspent output, a user (or group of users) must fulfill the - "…three of these four people must sign." - "…either Bob must sign, or both Tom and Sylvia must sign." -For details, see -`the documentation about conditions in the IPDB Transaction Spec `_. +For details, see the section about conditions +in the `IPDB Transaction Spec `_. Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output. diff --git a/docs/root/source/smart-contracts.rst b/docs/root/source/smart-contracts.rst index 143d6dfe..18b8a21f 100644 --- a/docs/root/source/smart-contracts.rst +++ b/docs/root/source/smart-contracts.rst @@ -15,4 +15,4 @@ Crypto-conditions can be quite complex. They can't include loops or recursion an .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the note about "owners" in the `IPDB Transaction Spec `_. diff --git a/docs/root/source/transaction-concepts.md b/docs/root/source/transaction-concepts.md index 02820ac5..93be77e4 100644 --- a/docs/root/source/transaction-concepts.md +++ b/docs/root/source/transaction-concepts.md @@ -30,14 +30,14 @@ Each output also has an associated condition: the condition that must be met BigchainDB supports a variety of conditions, a subset of the [Interledger Protocol (ILP)](https://interledger.org/) crypto-conditions. For details, see -[the documentation about conditions in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-components/conditions.html). +the section about conditions in the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). Each output also has a list of all the public keys associated with the conditions on that output. Loosely speaking, that list might be interpreted as the list of "owners." A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. -See the [note about "owners" in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/ownership.html). +See the note about "owners" in the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). A CREATE transaction must be signed by all the owners. (If you're looking for that signature, @@ -87,7 +87,7 @@ things. We documented those things in a post on *The BigchainDB Blog*: ["What is a Valid Transaction in BigchainDB?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598) (Note: That post was about BigchainDB Server v1.0.0.) -The [IPDB Transaction Spec documents the conditions for a transaction to be valid](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-validation.html). +The [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec) documents the conditions for a transaction to be valid. ## Example Transactions diff --git a/docs/server/source/appendices/cryptography.rst b/docs/server/source/appendices/cryptography.rst index a03e073b..9347fa25 100644 --- a/docs/server/source/appendices/cryptography.rst +++ b/docs/server/source/appendices/cryptography.rst @@ -1,9 +1,8 @@ Cryptography ============ -See `the IPDB Transaction Spec -`_, -especially the pages about: +See the `IPDB Transaction Spec `_, +especially the sections about: - Cryptographic Hashes - Cryptographic Keys & Signatures diff --git a/docs/server/source/appendices/json-serialization.rst b/docs/server/source/appendices/json-serialization.rst index fc67080d..e5b7ac15 100644 --- a/docs/server/source/appendices/json-serialization.rst +++ b/docs/server/source/appendices/json-serialization.rst @@ -1,6 +1,5 @@ JSON Serialization ================== -See the page about JSON Serialization & Deserialization -in `the IPDB Transaction Spec -`_. +See the section about JSON Serialization & Deserialization +in the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/asset-model.rst b/docs/server/source/data-models/asset-model.rst index 2046422f..1d6bc13e 100644 --- a/docs/server/source/data-models/asset-model.rst +++ b/docs/server/source/data-models/asset-model.rst @@ -1,5 +1,4 @@ The Asset Model =============== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index 0537bfd8..da9690f4 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -32,8 +32,8 @@ To compute it, 1) construct an :term:`associative array` ``d`` containing ``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, ``block.voters``, and their values. 2) compute ``id = hash_of_aa(d)``. There's pseudocode for the ``hash_of_aa()`` function -in the `IPDB Transaction Spec page about cryptographic hashes -`_. +in the `IPDB Transaction Spec `_ +section about cryptographic hashes. The result (``id``) is a string: the block ID. An example is ``"b60adf655932bf47ef58c0bfb2dd276d4795b94346b36cbb477e10d7eb02cea8"`` @@ -56,8 +56,8 @@ A list of the :ref:`transactions ` included in the block. The public key of the node that created the block. It's a string. -See the `IPDB Transaction Spec page about cryptographic keys & signatures -`_. +See the `IPDB Transaction Spec `_ +section about cryptographic keys & signatures. **block.voters** @@ -82,8 +82,8 @@ To compute that: where ``private_key`` is the node's private key (i.e. ``node_pubkey`` and ``private_key`` are a key pair). There's pseudocode for the ``sig_of_aa()`` function - on `the IPDB Transaction Spec page about cryptographic keys and signatures - `_. + on the `IPDB Transaction Spec `_ + section about cryptographic keys and signatures. .. note:: diff --git a/docs/server/source/data-models/conditions.rst b/docs/server/source/data-models/conditions.rst index 4eb8898a..4e299b9f 100644 --- a/docs/server/source/data-models/conditions.rst +++ b/docs/server/source/data-models/conditions.rst @@ -1,5 +1,4 @@ Conditions ========== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index f460d216..8f4e8762 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -1,5 +1,4 @@ Inputs and Outputs ================== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 6e7dadc9..8b43ff01 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -1,8 +1,7 @@ The Transaction Model ===================== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. The Transaction Schema @@ -11,7 +10,8 @@ The Transaction Schema BigchainDB checks all transactions (JSON documents) against a formal schema defined in some `JSON Schema `_ files. -Those files are part of the IPDB Transaction Spec. +Those files are part of the +`IPDB Transaction Spec `_. Their official source is the ``tx_schema/`` directory in the `ipdb/ipdb-tx-spec repository on GitHub `_, diff --git a/docs/server/source/data-models/vote-model.rst b/docs/server/source/data-models/vote-model.rst index 7f428f56..1068dc0c 100644 --- a/docs/server/source/data-models/vote-model.rst +++ b/docs/server/source/data-models/vote-model.rst @@ -35,8 +35,8 @@ The JSON Keys in a Vote The public key of the node which cast this vote. It's a string. For more information about public keys, -see the `IPDB Transaction Spec page about cryptographic keys and signatures -`_. +see the `IPDB Transaction Spec `_ +section about cryptographic keys and signatures. **vote.voting_for_block** @@ -92,9 +92,8 @@ To compute that: #. Compute ``signature = sig_of_aa(d, private_key)``, where ``private_key`` is the node's private key (i.e. ``node_pubkey`` and ``private_key`` are a key pair). There's pseudocode for the ``sig_of_aa()`` function - on `the IPDB Transaction Spec page about cryptographic keys and signatures - `_. - + on the `IPDB Transaction Spec `_ + section about cryptographic keys and signatures. The Vote Schema --------------- From 1611e6e04b225336cef4c02c007bedfedad39e34 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Feb 2018 15:24:19 +0100 Subject: [PATCH 02/28] Mount other volumes for docker-compose dev --- docker-compose.tendermint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 7c4489aa..0796b562 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -18,6 +18,11 @@ services: volumes: - ./bigchaindb:/usr/src/app/bigchaindb - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb From 87d74a704bdd64eefa3a7c55a80cb7c76a51c6db Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 12 Feb 2018 18:01:46 +0100 Subject: [PATCH 03/28] Problem: .pytest_cache is tracked by git Solution: Put .pytest_cache/ in the .gitignore file. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5397ca64..86fa44c1 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ htmlcov/ .coverage .coverage.* .cache +.pytest_cache/ nosetests.xml coverage.xml *.cover From b8a33e0f3080545c64280b8fa6cc3b6cb004f6d1 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 12 Feb 2018 19:25:08 +0100 Subject: [PATCH 04/28] Problem: tmdata tracked files change in dev env Solution: Only track config.toml and generate the other files such as genesis.json and priv_validator.json in the container via `tendermint init` Related issue: #1997 --- docker-compose.tendermint.yml | 4 ++-- tmdata/genesis.json | 1 - tmdata/priv_validator.json | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 tmdata/genesis.json delete mode 100644 tmdata/priv_validator.json diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 0796b562..dc798723 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -38,9 +38,9 @@ services: tendermint: image: tendermint/tendermint:0.13 volumes: - - ./tmdata:/tendermint + - ./tmdata/config.toml:/tendermint/config.toml entrypoint: '' - command: bash -c "tendermint unsafe_reset_all && tendermint node" + command: bash -c "tendermint init && tendermint node" curl-client: image: appropriate/curl command: /bin/sh -c "curl http://tendermint:46657/abci_query && curl http://bdb:9984/" diff --git a/tmdata/genesis.json b/tmdata/genesis.json deleted file mode 100644 index 1c5ae703..00000000 --- a/tmdata/genesis.json +++ /dev/null @@ -1 +0,0 @@ -{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-EhS6zg","validators":[{"pub_key":{"type":"ed25519","data":"C3E96823EB67401C5B794F4100CEA04B745C29A0707979485EAE4F3C1A7D8583"},"power":10,"name":""}],"app_hash":""} \ No newline at end of file diff --git a/tmdata/priv_validator.json b/tmdata/priv_validator.json deleted file mode 100644 index d48911de..00000000 --- a/tmdata/priv_validator.json +++ /dev/null @@ -1 +0,0 @@ -{"address":"2571CF34499E67BED107A6FE73418667A4461F6A","pub_key":{"type":"ed25519","data":"C3E96823EB67401C5B794F4100CEA04B745C29A0707979485EAE4F3C1A7D8583"},"last_height":1,"last_round":0,"last_step":3,"last_signature":{"type":"ed25519","data":"C9C16BEB7C71DC6FFDFE06973BEFDD995BD12F5A3B124F387E037C6E6F2CD913FA084495DEDEFABE5C52C385C8B909FDC0EB9BC15F3499B97C7693DCD1F82004"},"last_signbytes":"7B22636861696E5F6964223A22746573742D636861696E2D456853367A67222C22766F7465223A7B22626C6F636B5F6964223A7B2268617368223A2239334634373641364642323431393131343434324232444135434339303539353443393337373830222C227061727473223A7B2268617368223A2236423034414544374631323742353534433133383435394439413445423130394242354232413431222C22746F74616C223A317D7D2C22686569676874223A312C22726F756E64223A302C2274797065223A327D7D","priv_key":{"type":"ed25519","data":"5431111D9D8833305509308708341EBA0AD7226F30DBDA43616B4A73F94F2369C3E96823EB67401C5B794F4100CEA04B745C29A0707979485EAE4F3C1A7D8583"}} \ No newline at end of file From c35e37be64c459318d267971cf2b49d18140a0f5 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 12 Feb 2018 19:31:45 +0100 Subject: [PATCH 05/28] Problem: Unpublished ports with docker-compose Solution: Publish the ports. Closes #2041. --- docker-compose.tendermint.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index dc798723..1354de55 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -34,12 +34,16 @@ services: TENDERMINT_PORT: 46657 ports: - "9984" + - "46658" command: bigchaindb -l DEBUG start tendermint: image: tendermint/tendermint:0.13 volumes: - ./tmdata/config.toml:/tendermint/config.toml entrypoint: '' + ports: + - "46656" + - "46657" command: bash -c "tendermint init && tendermint node" curl-client: image: appropriate/curl From 4157244df7f41fc7143b8c903417279050199552 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 13 Feb 2018 14:09:41 +0100 Subject: [PATCH 06/28] Problem: statsd config is no longer supported Solution: remove monitoring code, completes #1138 --- bigchaindb/__init__.py | 3 - bigchaindb/core.py | 3 - bigchaindb/pipelines/block.py | 2 - bigchaindb/pipelines/vote.py | 1 - bigchaindb/web/views/transactions.py | 1 - docker-compose.benchmark.yml | 10 -- .../source/server-reference/configuration.md | 35 +---- scripts/benchmarks/README.md | 40 ------ scripts/benchmarks/create_thoughtput.py | 133 ------------------ setup.py | 1 - tests/test_config_utils.py | 1 - tests/test_core.py | 1 - 12 files changed, 4 insertions(+), 227 deletions(-) delete mode 100644 scripts/benchmarks/README.md delete mode 100644 scripts/benchmarks/create_thoughtput.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 34c981b7..a654f86a 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -123,9 +123,6 @@ config = { 'granular_levels': {}, 'port': log_config['root']['port'] }, - 'graphite': { - 'host': os.environ.get('BIGCHAINDB_GRAPHITE_HOST', 'localhost'), - }, } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 4fb84722..dab88045 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,5 +1,4 @@ import random -import statsd from time import time from bigchaindb import exceptions as core_exceptions @@ -72,8 +71,6 @@ class Bigchain(object): if not self.me or not self.me_private: raise exceptions.KeypairNotFoundException() - self.statsd = statsd.StatsClient(bigchaindb.config['graphite']['host']) - federation = property(lambda self: set(self.nodes_except_me + [self.me])) """ Set of federation member public keys """ diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index 4d482255..8c90313a 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -126,8 +126,6 @@ class BlockPipeline: logger.info('Write new block %s with %s transactions', block.id, len(block.transactions)) self.bigchain.write_block(block) - self.bigchain.statsd.incr('pipelines.block.throughput', - len(block.transactions)) return block def delete_tx(self, block): diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index f6959da0..fe19da58 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -149,7 +149,6 @@ class Vote: logger.info("Voting '%s' for block %s", validity, vote['vote']['voting_for_block']) self.bigchain.write_vote(vote) - self.bigchain.statsd.incr('pipelines.vote.throughput', num_tx) return vote diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index c746f2ad..ae753208 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -82,7 +82,6 @@ class TransactionListApi(Resource): ) with pool() as bigchain: - bigchain.statsd.incr('web.tx.post') try: bigchain.validate_transaction(tx_obj) except ValidationError as e: diff --git a/docker-compose.benchmark.yml b/docker-compose.benchmark.yml index c7319040..fa71875f 100644 --- a/docker-compose.benchmark.yml +++ b/docker-compose.benchmark.yml @@ -22,16 +22,6 @@ services: BIGCHAINDB_DATABASE_HOST: mdb BIGCHAINDB_DATABASE_PORT: 27017 BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 - BIGCHAINDB_GRAPHITE_HOST: graphite ports: - "9984" command: bigchaindb start - - graphite: - image: hopsoft/graphite-statsd - ports: - - "2003-2004" - - "2023-2024" - - "8125/udp" - - "8126" - - "80" diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index b7c5526e..e6f4c296 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -48,7 +48,6 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_DATABASE_KEYFILE`
`BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE`
`BIGCHAINDB_DATABASE_CRLFILE`
-`BIGCHAINDB_GRAPHITE_HOST`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -217,7 +216,7 @@ export BIGCHAINDB_SERVER_WORKERS=5 ## wsserver.scheme, wsserver.host and wsserver.port These settings are for the -[aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), +[aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), which is used to serve the [WebSocket Event Stream API](../websocket-event-stream-api.html). `wsserver.scheme` should be either `"ws"` or `"wss"` @@ -360,8 +359,8 @@ The full paths to the files where logs and error logs should be written to. } ``` -**Defaults to**: - +**Defaults to**: + * `"~/bigchaindb.log"` * `"~/bigchaindb-errors.log"` @@ -383,7 +382,7 @@ For example if we consider the log file setting: ``` logs would always be written to `bigchain.log`. Each time the file -`bigchain.log` reaches 200 MB it would be closed and renamed +`bigchain.log` reaches 200 MB it would be closed and renamed `bigchain.log.1`. If `bigchain.log.1` and `bigchain.log.2` already exist they would be renamed `bigchain.log.2` and `bigchain.log.3`. This pattern would be applied up to `bigchain.log.5` after which `bigchain.log.5` would be @@ -550,29 +549,3 @@ The port number at which the logging server should listen. ``` **Defaults to**: `9020` - - -## graphite.host - -The host name or IP address of a server listening for statsd events on UDP -port 8125. This defaults to `localhost`, and if no statsd collector is running, -the events are simply dropped by the operating system. - -**Example using environment variables** -```text -export BIGCHAINDB_GRAPHITE_HOST=10.0.0.5 -``` - -**Example config file snippet** -```js -"graphite": { - "host": "10.0.0.5" -} -``` - -**Default values (from a config file)** -```js -"graphite": { - "host": "localhost" -} -``` diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md deleted file mode 100644 index befe2400..00000000 --- a/scripts/benchmarks/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Benchmarks - -## CREATE transaction throughput - -This is a measurement of the throughput of CREATE transactions through the entire -pipeline, ie, the web frontend, block creation, and block validation, where the -output of the measurement is transactions per second. - -The benchmark runs for a fixed period of time and makes metrics available via -a graphite interface. - -### Running the benchmark - -Dependencies: - -* Python 3.5+ -* docker-compose 1.8.0+ -* docker 1.12+ - -To start: - - $ python3 scripts/benchmarks/create_thoughtput.py - -To start using a separate namespace for docker-compose: - - $ COMPOSE_PROJECT_NAME=somename python3 scripts/benchmarks/create_thoughtput.py - -### Results - -A test was run on AWS with the following instance configuration: - -* Ubuntu Server 16.04 (ami-060cde69) -* 32 core compute optimized (c3.8xlarge) -* 100gb root volume (300/3000 IOPS) - -The server received and validated over 800 transactions per second: - -![BigchainDB transaction throughput](https://cloud.githubusercontent.com/assets/125019/26688641/85d56d1e-46f3-11e7-8148-bf3bc8c54c33.png) - -For more information on how the benchmark was run, the abridged session buffer [is available](https://gist.github.com/libscott/8a37c5e134b2d55cfb55082b1cd85a02). diff --git a/scripts/benchmarks/create_thoughtput.py b/scripts/benchmarks/create_thoughtput.py deleted file mode 100644 index 2fe8c557..00000000 --- a/scripts/benchmarks/create_thoughtput.py +++ /dev/null @@ -1,133 +0,0 @@ -import sys -import math -import time -import requests -import subprocess -import multiprocessing - - -def main(): - cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d mdb') - cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d bdb') - cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d graphite') - - out = cmd('docker-compose -f benchmark.yml port graphite 80', capture=True) - graphite_web = 'http://localhost:%s/' % out.strip().split(':')[1] - print('Graphite web interface at: ' + graphite_web) - - start = time.time() - - cmd('docker-compose -f docker-compose.yml -f benchmark.yml exec bdb python %s load' % sys.argv[0]) - - mins = math.ceil((time.time() - start) / 60) + 1 - - graph_url = graphite_web + 'render/?width=900&height=600&_salt=1495462891.335&target=stats.pipelines.block.throughput&target=stats.pipelines.vote.throughput&target=stats.web.tx.post&from=-%sminutes' % mins # noqa - - print(graph_url) - - -def load(): - from bigchaindb.core import Bigchain - from bigchaindb.common.crypto import generate_key_pair - from bigchaindb.common.transaction import Transaction - - def transactions(): - priv, pub = generate_key_pair() - tx = Transaction.create([pub], [([pub], 1)]) - while True: - i = yield tx.to_dict() - tx.asset = {'data': {'n': i}} - tx.sign([priv]) - - def wait_for_up(): - print('Waiting for server to start... ', end='') - while True: - try: - requests.get('http://localhost:9984/') - break - except requests.ConnectionError: - time.sleep(0.1) - print('Ok') - - def post_txs(): - txs = transactions() - txs.send(None) - try: - with requests.Session() as session: - while True: - i = tx_queue.get() - if i is None: - break - tx = txs.send(i) - res = session.post('http://localhost:9984/api/v1/transactions/', json=tx) - assert res.status_code == 202 - except KeyboardInterrupt: - pass - - wait_for_up() - num_clients = 30 - test_time = 60 - tx_queue = multiprocessing.Queue(maxsize=num_clients) - txn = 0 - b = Bigchain() - - start_time = time.time() - - for i in range(num_clients): - multiprocessing.Process(target=post_txs).start() - - print('Sending transactions') - while time.time() - start_time < test_time: - # Post 500 transactions to the server - for i in range(500): - tx_queue.put(txn) - txn += 1 - print(txn) - while True: - # Wait for the server to reduce the backlog to below - # 10000 transactions. The expectation is that 10000 transactions - # will not be processed faster than a further 500 transactions can - # be posted, but nonetheless will be processed within a few seconds. - # This keeps the test from running on and keeps the transactions from - # being considered stale. - count = b.connection.db.backlog.count() - if count > 10000: - time.sleep(0.2) - else: - break - - for i in range(num_clients): - tx_queue.put(None) - - print('Waiting to clear backlog') - while True: - bl = b.connection.db.backlog.count() - if bl == 0: - break - print(bl) - time.sleep(1) - - print('Waiting for all votes to come in') - while True: - blocks = b.connection.db.bigchain.count() - votes = b.connection.db.votes.count() - if blocks == votes + 1: - break - print('%s blocks, %s votes' % (blocks, votes)) - time.sleep(3) - - print('Finished') - - -def cmd(command, capture=False): - stdout = subprocess.PIPE if capture else None - args = ['bash', '-c', command] - proc = subprocess.Popen(args, stdout=stdout) - assert not proc.wait() - return capture and proc.stdout.read().decode() - - -if sys.argv[1:] == ['load']: - load() -else: - main() diff --git a/setup.py b/setup.py index d60dc346..8079580d 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,6 @@ install_requires = [ 'pyyaml~=3.12', 'aiohttp~=2.3', 'python-rapidjson-schema==0.1.1', - 'statsd==3.2.1', 'abci~=0.3.0', ] diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index dd6aef4a..f156c871 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -304,7 +304,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request, ssl_con 'granular_levels': {}, 'port': 9020 }, - 'graphite': {'host': 'localhost'}, } diff --git a/tests/test_core.py b/tests/test_core.py index 7a369bdd..b18c56d3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -24,7 +24,6 @@ def config(request, monkeypatch): 'keyring': [], 'CONFIGURED': True, 'backlog_reassign_delay': 30, - 'graphite': {'host': 'localhost'}, } monkeypatch.setattr('bigchaindb.config', config) From 6e3f9e8f54bc380bdd14de71895b146dbde9b15e Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 14 Feb 2018 10:19:04 +0100 Subject: [PATCH 07/28] Docker compose for travis (#2039) * Replace double quotes with single quotes (flake8) * Test event subscriber to tendermint via ws * Problem: Tendermint is not part of stack for CI Solution: Add Tendermint to stack for CI. For simplicity's sake docker-compose is being used. --- .ci/travis-after-success.sh | 2 +- .ci/travis-before-install.sh | 13 +++++++---- .ci/travis-before-script.sh | 25 ++------------------ .ci/travis-install.sh | 2 +- .ci/travis_script.sh | 12 +--------- .travis.yml | 9 +++++--- bigchaindb/tendermint/event_stream.py | 10 ++++---- compose/travis/Dockerfile | 29 +++++++++++++++++++++++ docker-compose.travis.yml | 33 +++++++++++++++++++++++++++ setup.py | 1 + tests/conftest.py | 26 +++++++++++++++++++++ tests/tendermint/test_core.py | 12 +++++----- tests/tendermint/test_event_stream.py | 23 ++++++++++++++++--- 13 files changed, 140 insertions(+), 57 deletions(-) create mode 100644 compose/travis/Dockerfile create mode 100644 docker-compose.travis.yml diff --git a/.ci/travis-after-success.sh b/.ci/travis-after-success.sh index 2fc659e6..bb00d7a0 100755 --- a/.ci/travis-after-success.sh +++ b/.ci/travis-after-success.sh @@ -3,5 +3,5 @@ set -e -x if [[ -z ${TOXENV} ]]; then - codecov + codecov -v fi diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh index 3757e908..28111c80 100755 --- a/.ci/travis-before-install.sh +++ b/.ci/travis-before-install.sh @@ -1,6 +1,11 @@ #!/bin/bash -apt-get update -qq -wget https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz -O /tmp/v1.1.0.tar.gz -tar -xvf /tmp/v1.1.0.tar.gz -cp -r $PWD/rapidjson-1.1.0/include/rapidjson /usr/include/ +if [[ -z ${TOXENV} ]]; then + sudo apt-get update + sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + + sudo rm /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + chmod +x docker-compose + sudo mv docker-compose /usr/local/bin +fi diff --git a/.ci/travis-before-script.sh b/.ci/travis-before-script.sh index 22a275b1..b3115b3f 100755 --- a/.ci/travis-before-script.sh +++ b/.ci/travis-before-script.sh @@ -2,27 +2,6 @@ set -e -x -if [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then - # Connect to MongoDB on port 27017 via a normal, unsecure connection if - # BIGCHAINDB_DATABASE_SSL is unset. - # It is unset in this case in .travis.yml. - docker pull mongo:3.4 - docker run -d --publish=27017:27017 --name mdb-without-ssl mongo:3.4 # --replSet=bigchain-rs -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then - # Connect to MongoDB on port 27017 via TLS/SSL connection if - # BIGCHAINDB_DATABASE_SSL is set. - # It is set to 'true' here in .travis.yml. Dummy certificates for testing - # are stored under bigchaindb/tests/backend/mongodb-ssl/certs/ directory. - docker pull mongo:3.4 - docker run -d \ - --name mdb-with-ssl \ - --publish=27017:27017 \ - --volume=${TRAVIS_BUILD_DIR}/tests/backend/mongodb-ssl/certs:/certs \ - mongo:3.4 \ - --sslMode=requireSSL \ - --sslCAFile=/certs/ca-chain.cert.pem \ - --sslCRLFile=/certs/crl.pem \ - --sslPEMKeyFile=/certs/local-mongo.pem +if [[ -z ${TOXENV} ]]; then + docker-compose -f docker-compose.travis.yml up -d bdb fi diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 097f81dc..4af4908d 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -7,6 +7,6 @@ pip install --upgrade pip if [[ -n ${TOXENV} ]]; then pip install --upgrade tox else - pip install .[test] + docker-compose -f docker-compose.travis.yml build --no-cache pip install --upgrade codecov fi diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh index b33c40cc..72535932 100755 --- a/.ci/travis_script.sh +++ b/.ci/travis_script.sh @@ -4,16 +4,6 @@ set -e -x if [[ -n ${TOXENV} ]]; then tox -e ${TOXENV} -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then - # Run the full suite of tests for MongoDB over an unsecure connection - pytest -sv --database-backend=localmongodb --cov=bigchaindb -m tendermint -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then - # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'. - pytest -sv --database-backend=localmongodb-ssl --cov=bigchaindb -m bdb_ssl else - # Run the full suite of tests for RethinkDB (the default backend when testing) - pytest -sv -m "serial" - pytest -sv --cov=bigchaindb -m "not serial" + docker-compose -f docker-compose.travis.yml run --rm --no-deps bdb pytest -v --cov=bigchaindb fi diff --git a/.travis.yml b/.travis.yml index d1b281dc..8211e652 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,12 @@ python: - 3.6 env: - - TOXENV=flake8 - - TOXENV=docsroot - - TOXENV=docsserver + global: + - DOCKER_COMPOSE_VERSION=1.19.0 + matrix: + - TOXENV=flake8 + - TOXENV=docsroot + - TOXENV=docsserver matrix: fast_finish: true diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index 765fb190..9db8f2d9 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -25,7 +25,7 @@ def connect_and_recv(event_queue): logger.info('Connected to tendermint ws server') - stream_id = "bigchaindb_stream_{}".format(gen_timestamp()) + stream_id = 'bigchaindb_stream_{}'.format(gen_timestamp()) yield from subscribe_events(ws, stream_id) while True: @@ -58,10 +58,10 @@ def process_event(event_queue, event, stream_id): @asyncio.coroutine def subscribe_events(ws, stream_id): payload = { - "method": "subscribe", - "jsonrpc": "2.0", - "params": ["NewBlock"], - "id": stream_id + 'method': 'subscribe', + 'jsonrpc': '2.0', + 'params': ['NewBlock'], + 'id': stream_id } yield from ws.send_str(json.dumps(payload)) diff --git a/compose/travis/Dockerfile b/compose/travis/Dockerfile new file mode 100644 index 00000000..1717f9c6 --- /dev/null +++ b/compose/travis/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.6 +LABEL maintainer "dev@bigchaindb.com" + +RUN apt-get update \ + && pip install -U pip \ + && apt-get autoremove \ + && apt-get clean + +ARG backend + +ENV PYTHONUNBUFFERED 0 + +ENV BIGCHAINDB_DATABASE_PORT 27017 +ENV BIGCHAINDB_DATABASE_BACKEND $backend +ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 +ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_SCHEME ws + +ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws + +ENV BIGCHAINDB_START_TENDERMINT 0 +ENV TENDERMINT_PORT 46657 + +RUN mkdir -p /usr/src/app +COPY . /usr/src/app/ +WORKDIR /usr/src/app +RUN pip install --no-cache-dir .[test] +RUN bigchaindb -y configure "$backend" diff --git a/docker-compose.travis.yml b/docker-compose.travis.yml new file mode 100644 index 00000000..7a6db3ea --- /dev/null +++ b/docker-compose.travis.yml @@ -0,0 +1,33 @@ +version: '3' + +services: + mdb: + image: mongo:3.4.3 + command: mongod + bdb: + depends_on: + - mdb + - tendermint + build: + context: . + dockerfile: ./compose/travis/Dockerfile + args: + backend: localmongodb + volumes: + - .:/usr/src/app/ + environment: + BIGCHAINDB_DATABASE_BACKEND: localmongodb + BIGCHAINDB_DATABASE_HOST: mdb + BIGCHAINDB_DATABASE_PORT: 27017 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 + BIGCHAINDB_START_TENDERMINT: 0 + TENDERMINT_HOST: tendermint + TENDERMINT_PORT: 46657 + command: bigchaindb start + tendermint: + image: tendermint/tendermint:0.13 + volumes: + - ./tmdata/config.toml:/tendermint/config.toml + entrypoint: '' + command: bash -c "tendermint init && tendermint node" diff --git a/setup.py b/setup.py index 8079580d..0b143dd2 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ tests_require = [ 'pytest-xdist', 'pytest-flask', 'pytest-aiohttp', + 'pytest-asyncio', 'tox', ] + docs_require diff --git a/tests/conftest.py b/tests/conftest.py index 90894491..88507072 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -499,6 +499,32 @@ def db_context(db_config, db_host, db_port, db_name, db_conn): ) +@pytest.fixture +def tendermint_host(): + return os.getenv('TENDERMINT_HOST', 'localhost') + + +@pytest.fixture +def tendermint_port(): + return int(os.getenv('TENDERMINT_PORT', 46657)) + + +@pytest.fixture +def tendermint_ws_url(tendermint_host, tendermint_port): + return f'ws://{tendermint_host}:{tendermint_port}/websocket' + + +@pytest.fixture +def tendermint_context(tendermint_host, tendermint_port, tendermint_ws_url): + TendermintContext = namedtuple( + 'TendermintContext', ('host', 'port', 'ws_url')) + return TendermintContext( + host=tendermint_host, + port=tendermint_port, + ws_url=tendermint_ws_url, + ) + + @pytest.fixture def mocked_setup_pub_logger(mocker): return mocker.patch( diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py index 56c871e6..7456f130 100644 --- a/tests/tendermint/test_core.py +++ b/tests/tendermint/test_core.py @@ -57,8 +57,8 @@ def test_deliver_tx__valid_create_updates_db(b): .sign([alice.private_key]) app = App(b) - app.init_chain(["ignore"]) - app.begin_block("ignore") + app.init_chain(['ignore']) + app.begin_block('ignore') result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_ok() @@ -81,8 +81,8 @@ def test_deliver_tx__double_spend_fails(b): .sign([alice.private_key]) app = App(b) - app.init_chain(["ignore"]) - app.begin_block("ignore") + app.init_chain(['ignore']) + app.begin_block('ignore') result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_ok() @@ -101,8 +101,8 @@ def test_deliver_transfer_tx__double_spend_fails(b): from bigchaindb.common.crypto import generate_key_pair app = App(b) - app.init_chain(["ignore"]) - app.begin_block("ignore") + app.init_chain(['ignore']) + app.begin_block('ignore') alice = generate_key_pair() bob = generate_key_pair() diff --git a/tests/tendermint/test_event_stream.py b/tests/tendermint/test_event_stream.py index da5c4c6f..345f653c 100644 --- a/tests/tendermint/test_event_stream.py +++ b/tests/tendermint/test_event_stream.py @@ -1,9 +1,13 @@ +import json from queue import Queue +from aiohttp import ClientSession import pytest -@pytest.mark.tendermint +pytestmark = pytest.mark.tendermint + + def test_process_event_new_block(): from bigchaindb.tendermint.event_stream import process_event @@ -33,7 +37,6 @@ def test_process_event_new_block(): assert not event_queue.empty() -@pytest.mark.tendermint def test_process_event_empty_block(): from bigchaindb.tendermint.event_stream import process_event @@ -52,7 +55,6 @@ def test_process_event_empty_block(): assert event_queue.empty() -@pytest.mark.tendermint def test_process_unknown_event(): from bigchaindb.tendermint.event_stream import process_event @@ -62,3 +64,18 @@ def test_process_unknown_event(): event_queue = Queue() process_event(event_queue, event, 'test_stream_id') assert event_queue.empty() + + +@pytest.mark.asyncio +async def test_subscribe_events(tendermint_ws_url): + from bigchaindb.tendermint.event_stream import subscribe_events + session = ClientSession() + ws = await session.ws_connect(tendermint_ws_url) + stream_id = 'bigchaindb_stream_01' + await subscribe_events(ws, stream_id) + msg = await ws.receive() + assert msg.data + msg_data_dict = json.loads(msg.data) + assert msg_data_dict['id'] == stream_id + assert msg_data_dict['jsonrpc'] == '2.0' + # TODO What else should be there? Right now, getting error. From 55073366ecd318fd6961bc8385c6baba07674615 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 14 Feb 2018 10:44:44 +0100 Subject: [PATCH 08/28] Problem: new env vars are not managed by the conf Solution: add `BIGCHAINDB_` to new tendermint vars. Reference: https://github.com/bigchaindb/bigchaindb/pull/2039#discussion_r167880795 --- bigchaindb/tendermint/event_stream.py | 4 ++-- bigchaindb/tendermint/lib.py | 9 ++++++--- compose/bigchaindb-server/Dockerfile | 2 +- compose/travis/Dockerfile | 2 +- docker-compose.network.yml | 8 ++++---- docker-compose.tendermint.yml | 4 ++-- docker-compose.travis.yml | 4 ++-- tests/conftest.py | 4 ++-- 8 files changed, 20 insertions(+), 17 deletions(-) diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index 9db8f2d9..2191924b 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -11,8 +11,8 @@ from bigchaindb.events import EventTypes, Event from bigchaindb.tendermint.utils import decode_transaction_base64 -HOST = getenv('TENDERMINT_HOST', 'localhost') -PORT = int(getenv('TENDERMINT_PORT', 46657)) +HOST = getenv('BIGCHAINDB_TENDERMINT_HOST', 'localhost') +PORT = int(getenv('BIGCHAINDB_TENDERMINT_PORT', 46657)) URL = f'ws://{HOST}:{PORT}/websocket' logger = logging.getLogger(__name__) diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 37788b51..d4e6647f 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -17,9 +17,12 @@ from bigchaindb import exceptions as core_exceptions logger = logging.getLogger(__name__) -TENDERMINT_HOST = getenv('TENDERMINT_HOST', 'localhost') -TENDERMINT_PORT = getenv('TENDERMINT_PORT', '46657') -ENDPOINT = 'http://{}:{}/'.format(TENDERMINT_HOST, TENDERMINT_PORT) +BIGCHAINDB_TENDERMINT_HOST = getenv('BIGCHAINDB_TENDERMINT_HOST', + 'localhost') +BIGCHAINDB_TENDERMINT_PORT = getenv('BIGCHAINDB_TENDERMINT_PORT', + '46657') +ENDPOINT = 'http://{}:{}/'.format(BIGCHAINDB_TENDERMINT_HOST, + BIGCHAINDB_TENDERMINT_PORT) MODE_LIST = ('broadcast_tx_async', 'broadcast_tx_sync', 'broadcast_tx_commit') diff --git a/compose/bigchaindb-server/Dockerfile b/compose/bigchaindb-server/Dockerfile index 4abd8af8..56c63e22 100644 --- a/compose/bigchaindb-server/Dockerfile +++ b/compose/bigchaindb-server/Dockerfile @@ -22,7 +22,7 @@ ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws ENV BIGCHAINDB_START_TENDERMINT 0 -ENV TENDERMINT_PORT 46657 +ENV BIGCHAINDB_TENDERMINT_PORT 46657 RUN mkdir -p /usr/src/app diff --git a/compose/travis/Dockerfile b/compose/travis/Dockerfile index 1717f9c6..7a634684 100644 --- a/compose/travis/Dockerfile +++ b/compose/travis/Dockerfile @@ -20,7 +20,7 @@ ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws ENV BIGCHAINDB_START_TENDERMINT 0 -ENV TENDERMINT_PORT 46657 +ENV BIGCHAINDB_TENDERMINT_PORT 46657 RUN mkdir -p /usr/src/app COPY . /usr/src/app/ diff --git a/docker-compose.network.yml b/docker-compose.network.yml index 38a4b5c4..eb2cdfa6 100644 --- a/docker-compose.network.yml +++ b/docker-compose.network.yml @@ -27,7 +27,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-one - TENDERMINT_HOST: tendermint-one + BIGCHAINDB_TENDERMINT_HOST: tendermint-one ports: - "9984" command: bigchaindb -l DEBUG start @@ -64,7 +64,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-two - TENDERMINT_HOST: tendermint-two + BIGCHAINDB_TENDERMINT_HOST: tendermint-two ports: - "9984" command: bigchaindb -l DEBUG start @@ -101,7 +101,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-three - TENDERMINT_HOST: tendermint-three + BIGCHAINDB_TENDERMINT_HOST: tendermint-three ports: - "9984" command: bigchaindb -l DEBUG start @@ -138,7 +138,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-four - TENDERMINT_HOST: tendermint-four + BIGCHAINDB_TENDERMINT_HOST: tendermint-four ports: - "9984" command: bigchaindb -l DEBUG start diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 1354de55..d43af4d9 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -30,8 +30,8 @@ services: BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 BIGCHAINDB_START_TENDERMINT: 0 - TENDERMINT_HOST: tendermint - TENDERMINT_PORT: 46657 + BIGCHAINDB_TENDERMINT_HOST: tendermint + BIGCHAINDB_TENDERMINT_PORT: 46657 ports: - "9984" - "46658" diff --git a/docker-compose.travis.yml b/docker-compose.travis.yml index 7a6db3ea..e8d1c583 100644 --- a/docker-compose.travis.yml +++ b/docker-compose.travis.yml @@ -22,8 +22,8 @@ services: BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 BIGCHAINDB_START_TENDERMINT: 0 - TENDERMINT_HOST: tendermint - TENDERMINT_PORT: 46657 + BIGCHAINDB_TENDERMINT_HOST: tendermint + BIGCHAINDB_TENDERMINT_PORT: 46657 command: bigchaindb start tendermint: image: tendermint/tendermint:0.13 diff --git a/tests/conftest.py b/tests/conftest.py index 88507072..5d486ad8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -501,12 +501,12 @@ def db_context(db_config, db_host, db_port, db_name, db_conn): @pytest.fixture def tendermint_host(): - return os.getenv('TENDERMINT_HOST', 'localhost') + return os.getenv('BIGCHAINDB_TENDERMINT_HOST', 'localhost') @pytest.fixture def tendermint_port(): - return int(os.getenv('TENDERMINT_PORT', 46657)) + return int(os.getenv('BIGCHAINDB_TENDERMINT_PORT', 46657)) @pytest.fixture From f483f9c97abf627981a79e58dccdcc60ed9a4fe3 Mon Sep 17 00:00:00 2001 From: kansi Date: Wed, 14 Feb 2018 18:54:03 +0530 Subject: [PATCH 09/28] Add build config for docs --- .readthedocs.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..87fa516b --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,5 @@ +build: + image: latest + +python: + version: 3.6 From e9521aad44766e1935ea50e7139341daf2b03d5c Mon Sep 17 00:00:00 2001 From: kansi Date: Wed, 14 Feb 2018 19:40:36 +0530 Subject: [PATCH 10/28] Added build yml for readthedocs --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 87fa516b..c8b8b04c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,3 +3,4 @@ build: python: version: 3.6 + pip_install: true From 161ccdda5d2c5c6f74798934cbccd73decb5a610 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 14 Feb 2018 15:37:19 +0100 Subject: [PATCH 11/28] Implement UTXO set backend (#2033) --- bigchaindb/backend/localmongodb/query.py | 33 +++++++ bigchaindb/backend/localmongodb/schema.py | 13 ++- bigchaindb/backend/query.py | 31 ++++++ tests/backend/localmongodb/conftest.py | 12 +++ tests/backend/localmongodb/test_queries.py | 68 +++++++++++++ tests/backend/localmongodb/test_schema.py | 110 +++++++++++++++++++++ tests/conftest.py | 41 ++++++++ tests/utils.py | 2 + 8 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 tests/backend/localmongodb/conftest.py create mode 100644 tests/backend/localmongodb/test_schema.py diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index e167f99e..8e4f4c81 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -199,3 +199,36 @@ def get_block_with_transaction(conn, txid): conn.collection('blocks') .find({'transactions': txid}, projection={'_id': False, 'height': True})) + + +@register_query(LocalMongoDBConnection) +def store_unspent_outputs(conn, *unspent_outputs): + try: + return conn.run( + conn.collection('utxos') + .insert_many(unspent_outputs, ordered=False)) + except DuplicateKeyError: + # TODO log warning at least + pass + + +@register_query(LocalMongoDBConnection) +def delete_unspent_outputs(conn, *unspent_outputs): + cursor = conn.run( + conn.collection('utxos').remove( + {'$or': [ + {'$and': [ + {'transaction_id': unspent_output['transaction_id']}, + {'output_index': unspent_output['output_index']} + ]} + for unspent_output in unspent_outputs + ]} + )) + return cursor + + +@register_query(LocalMongoDBConnection) +def get_unspent_outputs(conn, *, query=None): + if query is None: + query = {} + return conn.run(conn.collection('utxos').find(query)) diff --git a/bigchaindb/backend/localmongodb/schema.py b/bigchaindb/backend/localmongodb/schema.py index 2d916856..5b152ee8 100644 --- a/bigchaindb/backend/localmongodb/schema.py +++ b/bigchaindb/backend/localmongodb/schema.py @@ -27,7 +27,7 @@ def create_database(conn, dbname): @register_schema(LocalMongoDBConnection) def create_tables(conn, dbname): - for table_name in ['transactions', 'assets', 'blocks', 'metadata']: + for table_name in ['transactions', 'utxos', 'assets', 'blocks', 'metadata']: logger.info('Create `%s` table.', table_name) # create the table # TODO: read and write concerns can be declared here @@ -40,6 +40,7 @@ def create_indexes(conn, dbname): create_assets_secondary_index(conn, dbname) create_blocks_secondary_index(conn, dbname) create_metadata_secondary_index(conn, dbname) + create_utxos_secondary_index(conn, dbname) @register_schema(LocalMongoDBConnection) @@ -99,3 +100,13 @@ def create_metadata_secondary_index(conn, dbname): # full text search index conn.conn[dbname]['metadata'].create_index([('$**', TEXT)], name='text') + + +def create_utxos_secondary_index(conn, dbname): + logger.info('Create `utxos` secondary index.') + + conn.conn[dbname]['utxos'].create_index( + [('transaction_id', ASCENDING), ('output_index', ASCENDING)], + name='utxo', + unique=True, + ) diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 96a27888..e2147e98 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -547,3 +547,34 @@ def store_block(conn, block): """ raise NotImplementedError + + +@singledispatch +def store_unspent_outputs(connection, unspent_outputs): + """Store unspent outputs in ``utxo_set`` table.""" + + raise NotImplementedError + + +@singledispatch +def delete_unspent_outputs(connection, unspent_outputs): + """Delete unspent outputs in ``utxo_set`` table.""" + + raise NotImplementedError + + +@singledispatch +def get_unspent_outputs(connection, *, query=None): + """Retrieves unspent outputs. + + Args: + query (dict): An optional parameter to filter the result set. + Defaults to ``None``, which means that all UTXO records + will be returned. + + Returns: + Generator yielding unspent outputs (UTXO set) according to the + given query. + """ + + raise NotImplementedError diff --git a/tests/backend/localmongodb/conftest.py b/tests/backend/localmongodb/conftest.py new file mode 100644 index 00000000..b51a0ebb --- /dev/null +++ b/tests/backend/localmongodb/conftest.py @@ -0,0 +1,12 @@ +from pymongo import MongoClient +from pytest import fixture + + +@fixture +def mongo_client(db_context): + return MongoClient(host=db_context.host, port=db_context.port) + + +@fixture +def utxo_collection(db_context, mongo_client): + return mongo_client[db_context.name].utxos diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index 75f5190c..768bfbf9 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -6,6 +6,23 @@ import pymongo pytestmark = [pytest.mark.tendermint, pytest.mark.localmongodb, pytest.mark.bdb] +@pytest.fixture +def dummy_unspent_outputs(): + return [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 1}, + {'transaction_id': 'b', 'output_index': 0}, + ] + + +@pytest.fixture +def utxoset(dummy_unspent_outputs, utxo_collection): + insert_res = utxo_collection.insert_many(deepcopy(dummy_unspent_outputs)) + assert insert_res.acknowledged + assert len(insert_res.inserted_ids) == 3 + return dummy_unspent_outputs, utxo_collection + + def test_get_txids_filtered(signed_create_tx, signed_transfer_tx): from bigchaindb.backend import connect, query from bigchaindb.models import Transaction @@ -180,3 +197,54 @@ def test_get_block(): block = dict(query.get_block(conn, 3)) assert block['height'] == 3 + + +def test_delete_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + unspent_outputs, utxo_collection = utxoset + delete_res = query.delete_unspent_outputs(db_context.conn, + *unspent_outputs[::2]) + assert delete_res['n'] == 2 + assert utxo_collection.find( + {'$or': [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'b', 'output_index': 0}, + ]} + ).count() == 0 + assert utxo_collection.find( + {'transaction_id': 'a', 'output_index': 1}).count() == 1 + + +def test_store_one_unspent_output(db_context, + unspent_output_1, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn, unspent_output_1) + assert res.acknowledged + assert len(res.inserted_ids) == 1 + assert utxo_collection.find( + {'transaction_id': unspent_output_1['transaction_id'], + 'output_index': unspent_output_1['output_index']} + ).count() == 1 + + +def test_store_many_unspent_outputs(db_context, + unspent_outputs, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn, *unspent_outputs) + assert res.acknowledged + assert len(res.inserted_ids) == 3 + assert utxo_collection.find( + {'transaction_id': unspent_outputs[0]['transaction_id']} + ).count() == 3 + + +def test_get_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + cursor = query.get_unspent_outputs(db_context.conn) + assert cursor.count() == 3 + retrieved_utxoset = list(cursor) + unspent_outputs, utxo_collection = utxoset + assert retrieved_utxoset == list(utxo_collection.find()) + for utxo in retrieved_utxoset: + del utxo['_id'] + assert retrieved_utxoset == unspent_outputs diff --git a/tests/backend/localmongodb/test_schema.py b/tests/backend/localmongodb/test_schema.py new file mode 100644 index 00000000..beee421c --- /dev/null +++ b/tests/backend/localmongodb/test_schema.py @@ -0,0 +1,110 @@ +import pytest + + +pytestmark = [pytest.mark.bdb, pytest.mark.tendermint] + + +def test_init_creates_db_tables_and_indexes(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend.schema import init_database + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # the db is set up by the fixture so we need to remove it + conn.conn.drop_database(dbname) + + init_database() + + collection_names = conn.conn[dbname].collection_names() + assert set(collection_names) == { + 'transactions', 'assets', 'metadata', 'blocks', 'utxos'} + + indexes = conn.conn[dbname]['assets'].index_information().keys() + assert set(indexes) == {'_id_', 'asset_id', 'text'} + + indexes = conn.conn[dbname]['transactions'].index_information().keys() + assert set(indexes) == { + '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} + + indexes = conn.conn[dbname]['blocks'].index_information().keys() + assert set(indexes) == {'_id_', 'height'} + + indexes = conn.conn[dbname]['utxos'].index_information().keys() + assert set(indexes) == {'_id_', 'utxo'} + + +def test_init_database_fails_if_db_exists(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend.schema import init_database + from bigchaindb.common import exceptions + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures + assert dbname in conn.conn.database_names() + + with pytest.raises(exceptions.DatabaseAlreadyExists): + init_database() + + +def test_create_tables(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures so we need to remove it + conn.conn.drop_database(dbname) + schema.create_database(conn, dbname) + schema.create_tables(conn, dbname) + + collection_names = conn.conn[dbname].collection_names() + assert set(collection_names) == { + 'transactions', 'assets', 'metadata', 'blocks', 'utxos'} + + +def test_create_secondary_indexes(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures so we need to remove it + conn.conn.drop_database(dbname) + schema.create_database(conn, dbname) + schema.create_tables(conn, dbname) + schema.create_indexes(conn, dbname) + + indexes = conn.conn[dbname]['assets'].index_information().keys() + assert set(indexes) == {'_id_', 'asset_id', 'text'} + + indexes = conn.conn[dbname]['transactions'].index_information().keys() + assert set(indexes) == { + '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} + + indexes = conn.conn[dbname]['blocks'].index_information().keys() + assert set(indexes) == {'_id_', 'height'} + + index_info = conn.conn[dbname]['utxos'].index_information() + assert set(index_info.keys()) == {'_id_', 'utxo'} + assert index_info['utxo']['unique'] + assert index_info['utxo']['key'] == [('transaction_id', 1), + ('output_index', 1)] + + +def test_drop(dummy_db): + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + assert dummy_db in conn.conn.database_names() + schema.drop_database(conn, dummy_db) + assert dummy_db not in conn.conn.database_names() diff --git a/tests/conftest.py b/tests/conftest.py index 5d486ad8..25fee5c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -617,3 +617,44 @@ def genesis_tx(b, user_pk): tx.operation = Transaction.GENESIS genesis_tx = tx.sign([b.me_private]) return genesis_tx + + +@pytest.fixture +def unspent_output_0(): + return { + 'amount': 1, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 0, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d' + } + + +@pytest.fixture +def unspent_output_1(): + return { + 'amount': 2, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 1, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + } + + +@pytest.fixture +def unspent_output_2(): + return { + 'amount': 3, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 2, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + } + + +@pytest.fixture +def unspent_outputs(unspent_output_0, unspent_output_1, unspent_output_2): + return unspent_output_0, unspent_output_1, unspent_output_2 diff --git a/tests/utils.py b/tests/utils.py index 50dedae9..d8784809 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,6 +46,7 @@ def flush_mongo_db(connection, dbname): connection.conn[dbname].votes.delete_many({}) connection.conn[dbname].assets.delete_many({}) connection.conn[dbname].metadata.delete_many({}) + connection.conn[dbname].utxos.delete_many({}) @flush_db.register(LocalMongoDBConnection) @@ -55,6 +56,7 @@ def flush_localmongo_db(connection, dbname): connection.conn[dbname].transactions.delete_many({}) connection.conn[dbname].assets.delete_many({}) connection.conn[dbname].metadata.delete_many({}) + connection.conn[dbname].utxos.delete_many({}) @singledispatch From bf2a1c6a60ca5f6797902368c42cd6c2e1033538 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 14 Feb 2018 10:20:56 +0100 Subject: [PATCH 12/28] Problem: older backends are no longer supported Solution: when running the command `bigchaindb configure`, configure for `localmongodb` only. --- bigchaindb/commands/bigchaindb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index ac81591b..3b2edb5a 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -271,7 +271,10 @@ def create_parser(): help='Prepare the config file ' 'and create the node keypair') config_parser.add_argument('backend', - choices=['rethinkdb', 'mongodb', 'localmongodb'], + choices=['localmongodb'], + default='localmongodb', + const='localmongodb', + nargs='?', help='The backend to use. It can be either ' 'rethinkdb or mongodb.') From 254035150d280cbd7aa275e1fabf564738bfb0a8 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 14 Feb 2018 10:22:10 +0100 Subject: [PATCH 13/28] Problem: default config uses env vars Solution: this is managed by the function `bigchaindb.config_utils::env_config`, we don't need to call `os.environ.get` for each value. --- bigchaindb/__init__.py | 79 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index a654f86a..a4006d3f 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -1,6 +1,5 @@ import copy import logging -import os from bigchaindb.log.configs import SUBSCRIBER_LOGGING_CONFIG as log_config @@ -10,9 +9,9 @@ from bigchaindb.log.configs import SUBSCRIBER_LOGGING_CONFIG as log_config _base_database_rethinkdb = { - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), + 'host': 'localhost', + 'port': 28015, + 'name': 'bigchain', } # The following variable is used by `bigchaindb configure` to @@ -27,53 +26,53 @@ _database_keys_map = { } _base_database_localmongodb = { - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET'), - 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), - 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') + 'host': 'localhost', + 'port': 27017, + 'name': 'bigchain', + 'replicaset': None, + 'login': None, + 'password': None, } _base_database_mongodb = { - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), - 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') + 'host': 'localhost', + 'port': 27017, + 'name': 'bigchain', + 'replicaset': 'bigchain-rs', + 'login': None, + 'password': None, } _database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + 'backend': 'rethinkdb', 'connection_timeout': 5000, 'max_tries': 3, } _database_rethinkdb.update(_base_database_rethinkdb) _database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'backend': 'mongodb', 'connection_timeout': 5000, 'max_tries': 3, - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), - 'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT'), - 'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE'), - 'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE'), - 'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE'), - 'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE') + 'ssl': False, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None, } _database_mongodb.update(_base_database_mongodb) _database_localmongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'localmongodb'), + 'backend': 'localmongodb', 'connection_timeout': 5000, 'max_tries': 3, - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), - 'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT'), - 'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE'), - 'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE'), - 'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE'), - 'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE') + 'ssl': False, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None, } _database_localmongodb.update(_base_database_localmongodb) @@ -87,22 +86,20 @@ config = { 'server': { # Note: this section supports all the Gunicorn settings: # - http://docs.gunicorn.org/en/stable/settings.html - 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', + 'bind': 'localhost:9984', 'loglevel': logging.getLevelName( log_config['handlers']['console']['level']).lower(), 'workers': None, # if none, the value will be cpu_count * 2 + 1 }, 'wsserver': { - 'scheme': os.environ.get('BIGCHAINDB_WSSERVER_SCHEME') or 'ws', - 'host': os.environ.get('BIGCHAINDB_WSSERVER_HOST') or 'localhost', - 'port': int(os.environ.get('BIGCHAINDB_WSSERVER_PORT', 9985)), - 'advertised_scheme': os.environ.get('BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME') or 'ws', - 'advertised_host': os.environ.get('BIGCHAINDB_WSSERVER_ADVERTISED_HOST') or 'localhost', - 'advertised_port': int(os.environ.get('BIGCHAINDB_WSSERVER_ADVERTISED_PORT', 9985)), + 'scheme': 'ws', + 'host': 'localhost', + 'port': 9985, + 'advertised_scheme': 'ws', + 'advertised_host': 'localhost', + 'advertised_port': 9985, }, - 'database': _database_map[ - os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') - ], + 'database': _database_map['localmongodb'], 'keypair': { 'public': None, 'private': None, From cd6095e854a825a23f63eabc2d119976820ff9e0 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 14 Feb 2018 15:30:21 +0100 Subject: [PATCH 14/28] Problem: test suite doesn't pick up env vars Solution: call `env_config` from the configuration fixture --- bigchaindb/__init__.py | 1 + tests/conftest.py | 1 + 2 files changed, 2 insertions(+) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index a4006d3f..673fe57f 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -99,6 +99,7 @@ config = { 'advertised_host': 'localhost', 'advertised_port': 9985, }, + # FIXME: hardcoding to localmongodb for now 'database': _database_map['localmongodb'], 'keypair': { 'public': None, diff --git a/tests/conftest.py b/tests/conftest.py index 25fee5c3..cf0299e3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -155,6 +155,7 @@ def _configure_bigchaindb(request, ssl_context): } } config['database']['name'] = test_db_name + config = config_utils.env_config(config) config_utils.set_config(config) From 63bd698b21b1e5d7fc24d9cf51c91feb10ebdf32 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 15 Feb 2018 11:07:16 +0100 Subject: [PATCH 15/28] changed constructed/allowed tx 'version' from '1.0' to '2.0' (#2048) - Created three new JSON Schema files for the v2.0 transaction schema, based on the v1.0 files. The only change is the allowed value of version (from "^1\\.0$" to "^2\\.0$"). - I didn't delete the v1.0 JSON Schema files because we might want those some day (to validate old transactions). - Updated the __init__.py in the directory with the JSON Schema files so that it now uses the version 2.0 JSON Schema files. - Updated all relevant tests. I only found one, in tests/validation/test_transaction_structure.py - Updated VERSION in common/transaction.py - Checked to make sure the example HTTP API docs show "version" with value "2.0". They do. - Updated the docs page about "The Transaction Model". It just points to the IPDB Transaction Spec. - If someone submits a transaction with "version" having value "1.0" then the error message comes from the JSON Schema checker. --- bigchaindb/common/schema/__init__.py | 2 +- .../schema/transaction_create_v2.0.yaml | 30 ++++ .../schema/transaction_transfer_v2.0.yaml | 29 ++++ .../common/schema/transaction_v2.0.yaml | 163 ++++++++++++++++++ bigchaindb/common/transaction.py | 2 +- .../source/data-models/transaction-model.rst | 18 -- .../validation/test_transaction_structure.py | 2 +- 7 files changed, 225 insertions(+), 21 deletions(-) create mode 100644 bigchaindb/common/schema/transaction_create_v2.0.yaml create mode 100644 bigchaindb/common/schema/transaction_transfer_v2.0.yaml create mode 100644 bigchaindb/common/schema/transaction_v2.0.yaml diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 0f4990dd..e64e5640 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -22,7 +22,7 @@ def _load_schema(name): return path, (schema, fast_schema) -TX_SCHEMA_VERSION = 'v1.0' +TX_SCHEMA_VERSION = 'v2.0' TX_SCHEMA_PATH, TX_SCHEMA_COMMON = _load_schema('transaction_' + TX_SCHEMA_VERSION) diff --git a/bigchaindb/common/schema/transaction_create_v2.0.yaml b/bigchaindb/common/schema/transaction_create_v2.0.yaml new file mode 100644 index 00000000..3d393347 --- /dev/null +++ b/bigchaindb/common/schema/transaction_create_v2.0.yaml @@ -0,0 +1,30 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - CREATE/GENESIS specific constraints +required: +- asset +- inputs +properties: + asset: + additionalProperties: false + properties: + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + required: + - data + inputs: + type: array + title: "Transaction inputs" + maxItems: 1 + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "null" diff --git a/bigchaindb/common/schema/transaction_transfer_v2.0.yaml b/bigchaindb/common/schema/transaction_transfer_v2.0.yaml new file mode 100644 index 00000000..538ec5e6 --- /dev/null +++ b/bigchaindb/common/schema/transaction_transfer_v2.0.yaml @@ -0,0 +1,29 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - TRANSFER specific properties +required: +- asset +properties: + asset: + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + required: + - id + inputs: + type: array + title: "Transaction inputs" + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "object" +definitions: + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string diff --git a/bigchaindb/common/schema/transaction_v2.0.yaml b/bigchaindb/common/schema/transaction_v2.0.yaml new file mode 100644 index 00000000..16cafd33 --- /dev/null +++ b/bigchaindb/common/schema/transaction_v2.0.yaml @@ -0,0 +1,163 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +additionalProperties: false +title: Transaction Schema +required: +- id +- inputs +- outputs +- operation +- metadata +- asset +- version +properties: + id: + anyOf: + - "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" + inputs: + type: array + title: "Transaction inputs" + items: + "$ref": "#/definitions/input" + outputs: + type: array + items: + "$ref": "#/definitions/output" + metadata: + "$ref": "#/definitions/metadata" + version: + type: string + pattern: "^2\\.0$" +definitions: + offset: + type: integer + minimum: 0 + base58: + pattern: "[1-9a-zA-Z^OIl]{43,44}" + type: string + public_keys: + anyOf: + - type: array + items: + "$ref": "#/definitions/base58" + - type: 'null' + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string + uuid4: + pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" + type: string + operation: + type: string + enum: + - CREATE + - TRANSFER + - GENESIS + asset: + type: object + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + output: + type: object + additionalProperties: false + required: + - amount + - condition + - public_keys + properties: + amount: + type: string + pattern: "^[0-9]{1,20}$" + condition: + type: object + additionalProperties: false + required: + - details + - uri + properties: + details: + "$ref": "#/definitions/condition_details" + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" + public_keys: + "$ref": "#/definitions/public_keys" + input: + type: "object" + additionalProperties: false + required: + - owners_before + - fulfillment + properties: + owners_before: + "$ref": "#/definitions/public_keys" + fulfillment: + anyOf: + - type: string + pattern: "^[a-zA-Z0-9_-]*$" + - "$ref": "#/definitions/condition_details" + fulfills: + anyOf: + - type: 'object' + additionalProperties: false + required: + - output_index + - transaction_id + properties: + output_index: + "$ref": "#/definitions/offset" + transaction_id: + "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + metadata: + anyOf: + - type: object + additionalProperties: true + minProperties: 1 + - type: 'null' + condition_details: + anyOf: + - type: object + additionalProperties: false + required: + - type + - public_key + properties: + type: + type: string + pattern: "^ed25519-sha-256$" + public_key: + "$ref": "#/definitions/base58" + - type: object + additionalProperties: false + required: + - type + - threshold + - subconditions + properties: + type: + type: "string" + pattern: "^threshold-sha-256$" + threshold: + type: integer + minimum: 1 + maximum: 100 + subconditions: + type: array + items: + "$ref": "#/definitions/condition_details" diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index e4738ca1..d2db0256 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -475,7 +475,7 @@ class Transaction(object): TRANSFER = 'TRANSFER' GENESIS = 'GENESIS' ALLOWED_OPERATIONS = (CREATE, TRANSFER, GENESIS) - VERSION = '1.0' + VERSION = '2.0' def __init__(self, operation, asset, inputs=None, outputs=None, metadata=None, version=None, hash_id=None): diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 8b43ff01..d8eafd12 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -2,21 +2,3 @@ The Transaction Model ===================== See the `IPDB Transaction Spec `_. - - -The Transaction Schema ----------------------- - -BigchainDB checks all transactions (JSON documents) -against a formal schema defined -in some `JSON Schema `_ files. -Those files are part of the -`IPDB Transaction Spec `_. -Their official source is the ``tx_schema/`` directory -in the `ipdb/ipdb-tx-spec repository on GitHub -`_, -but BigchainDB Server uses copies of those files; -those copies can be found -in the ``bigchaindb/common/schema/`` directory -in the `bigchaindb/bigchaindb repository on GitHub -`_. diff --git a/tests/validation/test_transaction_structure.py b/tests/validation/test_transaction_structure.py index 5b128003..35f86cd1 100644 --- a/tests/validation/test_transaction_structure.py +++ b/tests/validation/test_transaction_structure.py @@ -237,7 +237,7 @@ def test_unsupported_condition_type(): # Version def test_validate_version(b, create_tx): - create_tx.version = '1.0' + create_tx.version = '2.0' create_tx.sign([b.me_private]) validate(create_tx) From 3542d018935f7b7270a6762dbcb169a3e17bdb3c Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 15 Feb 2018 16:19:13 +0100 Subject: [PATCH 16/28] Update tx version (#2054) * Problem: common/test_transaction.py is excluded Solution: add marker to file * Problem: fixture tx version is outdated Solution: update the version --- tests/common/conftest.py | 2 +- tests/common/test_transaction.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 97cfcfb3..ae371adb 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -343,7 +343,7 @@ def tri_state_transaction(request): 'uri': 'ni:///sha-256;49C5UWNODwtcINxLgLc90bMCFqCymFYONGEmV4a0sG4?fpt=ed25519-sha-256&cost=131072'}, 'public_keys': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] }], - 'version': '1.0' + 'version': '2.0' } tx['id'] = request.param['id'] tx['inputs'][0]['fulfillment'] = request.param['fulfillment'] diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 39048e19..16410dcc 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -8,6 +8,8 @@ from base58 import b58encode, b58decode from pytest import mark, raises from sha3 import sha3_256 +pytestmark = mark.tendermint + def test_input_serialization(ffill_uri, user_pub): from bigchaindb.common.transaction import Input From 46a2909c4d5e8c8d0087df86944018c70b0689c6 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 15 Feb 2018 17:34:51 +0100 Subject: [PATCH 17/28] Remove 2 refs/links to the deprecated Transaction CLI tool --- docs/root/source/index.rst | 3 --- docs/server/source/drivers-clients/index.rst | 6 ------ 2 files changed, 9 deletions(-) diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst index 433075c1..50c4674c 100644 --- a/docs/root/source/index.rst +++ b/docs/root/source/index.rst @@ -60,9 +60,6 @@ At a high level, one can communicate with a BigchainDB cluster (set of nodes) us
JavaScript Driver Docs
-
- Command Line Transaction Tool -
Server Docs
diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 00250f60..b124a598 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -6,12 +6,6 @@ Libraries and Tools Maintained by the BigchainDB Team * `Python Driver `_ * `JavaScript / Node.js Driver `_ -* `The Transaction CLI `_ is - a command-line interface for building BigchainDB transactions. - You may be able to call it from inside the language of - your choice, and then use :ref:`the HTTP API ` - to post transactions. - Community-Driven Libraries and Tools ------------------------------------ From 1f537b1557eb353652167ef7aecaf5921fc5b517 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 15 Feb 2018 18:23:29 +0100 Subject: [PATCH 18/28] Problem: to get tx utxoset & inputs is cumbersome (#2055) Solution: add properties to more easily get the utxoset and inputs of a transaction --- bigchaindb/common/transaction.py | 45 ++++++++++++++++++++++++++++++++ tests/common/test_transaction.py | 40 ++++++++++++++++++++++++++++ tests/conftest.py | 16 ++++++++++++ 3 files changed, 101 insertions(+) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index d2db0256..13af253c 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1,7 +1,12 @@ """Transaction related models to parse and construct transaction payloads. +Attributes: + UnspentOutput (namedtuple): Object holding the information + representing an unspent output. + """ +from collections import namedtuple from copy import deepcopy from functools import reduce @@ -19,6 +24,17 @@ from bigchaindb.common.exceptions import (KeypairMismatchException, from bigchaindb.common.utils import serialize +UnspentOutput = namedtuple( + 'UnspentOutput', ( + 'transaction_id', + 'output_index', + 'amount', + 'asset_id', + 'condition_uri', + ) +) + + class Input(object): """A Input is used to spend assets locked by an Output. @@ -532,6 +548,35 @@ class Transaction(object): self.metadata = metadata self._id = hash_id + @property + def unspent_outputs(self): + """UnspentOutput: The outputs of this transaction, in a data + structure containing relevant information for storing them in + a UTXO set, and performing validation. + """ + if self.operation == Transaction.CREATE: + self._asset_id = self._id + elif self.operation == Transaction.TRANSFER: + self._asset_id = self.asset['id'] + return (UnspentOutput( + transaction_id=self._id, + output_index=output_index, + amount=output.amount, + asset_id=self._asset_id, + condition_uri=output.fulfillment.condition_uri, + ) for output_index, output in enumerate(self.outputs)) + + @property + def spent_outputs(self): + """tuple of :obj:`dict`: Inputs of this transaction. Each input + is represented as a dictionary containing a transaction id and + output index. + """ + return ( + input_.fulfills.to_dict() + for input_ in self.inputs if input_.fulfills + ) + @property def serialized(self): return Transaction._to_str(self.to_dict()) diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 16410dcc..2617197d 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -5,6 +5,7 @@ import json from copy import deepcopy from base58 import b58encode, b58decode +from cryptoconditions import Ed25519Sha256 from pytest import mark, raises from sha3 import sha3_256 @@ -1007,3 +1008,42 @@ def test_output_from_dict_invalid_amount(user_output): out['amount'] = 'a' with raises(AmountError): Output.from_dict(out) + + +def test_unspent_outputs_property(merlin, alice, bob, carol): + from bigchaindb.common.transaction import Transaction + tx = Transaction.create( + [merlin.public_key], + [([alice.public_key], 1), + ([bob.public_key], 2), + ([carol.public_key], 3)], + asset={'hash': '06e47bcf9084f7ecfd2a2a2ad275444a'}, + ).sign([merlin.private_key]) + unspent_outputs = list(tx.unspent_outputs) + assert len(unspent_outputs) == 3 + assert all(utxo.transaction_id == tx.id for utxo in unspent_outputs) + assert all(utxo.asset_id == tx.id for utxo in unspent_outputs) + assert all( + utxo.output_index == i for i, utxo in enumerate(unspent_outputs)) + unspent_output_0 = unspent_outputs[0] + assert unspent_output_0.amount == 1 + assert unspent_output_0.condition_uri == Ed25519Sha256( + public_key=b58decode(alice.public_key)).condition_uri + unspent_output_1 = unspent_outputs[1] + assert unspent_output_1.amount == 2 + assert unspent_output_1.condition_uri == Ed25519Sha256( + public_key=b58decode(bob.public_key)).condition_uri + unspent_output_2 = unspent_outputs[2] + assert unspent_output_2.amount == 3 + assert unspent_output_2.condition_uri == Ed25519Sha256( + public_key=b58decode(carol.public_key)).condition_uri + + +def test_spent_outputs_property(signed_transfer_tx): + spent_outputs = list(signed_transfer_tx.spent_outputs) + tx = signed_transfer_tx.to_dict() + assert len(spent_outputs) == 1 + spent_output = spent_outputs[0] + assert spent_output['transaction_id'] == tx['inputs'][0]['fulfills']['transaction_id'] + assert spent_output['output_index'] == tx['inputs'][0]['fulfills']['output_index'] + # assert spent_output._asdict() == tx['inputs'][0]['fulfills'] diff --git a/tests/conftest.py b/tests/conftest.py index cf0299e3..4828d5d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -310,6 +310,22 @@ def carol_pubkey(carol): return carol.public_key +@pytest.fixture +def merlin(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def merlin_privkey(merlin): + return merlin.private_key + + +@pytest.fixture +def merlin_pubkey(merlin): + return merlin.public_key + + @pytest.fixture def b(): from bigchaindb import Bigchain From 6cf86be0335cb49cce3fdf59341a0ac80837fcb2 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 16 Feb 2018 18:42:06 +0530 Subject: [PATCH 19/28] Remove location header from post txn response --- bigchaindb/web/views/transactions.py | 9 --------- docs/server/generate_http_server_api_documentation.py | 1 - docs/server/source/http-client-server-api.rst | 10 +++------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index ae753208..783d020a 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -95,13 +95,4 @@ class TransactionListApi(Resource): response = jsonify(tx) response.status_code = 202 - # NOTE: According to W3C, sending a relative URI is not allowed in the - # Location Header: - # - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html - # - # Flask is autocorrecting relative URIs. With the following command, - # we're able to prevent this. - response.autocorrect_location_header = False - status_monitor = '../statuses?transaction_id={}'.format(tx_obj.id) - response.headers['Location'] = status_monitor return response diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 7e683bb6..bc12f913 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -69,7 +69,6 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ HTTP/1.1 202 Accepted -Location: ../statuses?transaction_id=%(txid)s Content-Type: application/json %(tx)s diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 68ad5d7c..1e8d03e1 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -159,15 +159,11 @@ Transactions :language: http .. note:: - If the server is returning a ``202`` HTTP status code, then the - transaction has been accepted for processing. To check the status of the - transaction, poll the link to the - :ref:`status monitor ` - provided in the ``Location`` header or listen to server's - :ref:`WebSocket Event Stream API `. + If the server is returning a ``202`` HTTP status code when ``mode=aysnc`` or ``mode=sync``, then the + transaction has been accepted for processing. The client can subscribe to + :ref:`WebSocket Event Stream API ` for listen for comitted transactions. :resheader Content-Type: ``application/json`` - :resheader Location: Relative link to a status monitor for the submitted transaction. :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. From 9ef2271a5d312cd212702d9204dc517ca9550cff Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Feb 2018 10:51:05 +0100 Subject: [PATCH 20/28] In HTTP API docs, removed all /statuses & /votes endpoints --- docs/server/source/http-client-server-api.rst | 96 ------------------- 1 file changed, 96 deletions(-) diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 1e8d03e1..76f1f4f1 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -287,71 +287,6 @@ unspent outputs. :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. -Statuses --------------------------------- - -.. http:get:: /api/v1/statuses - - Get the status of an asynchronously written transaction or block by their id. - - :query string transaction_id: transaction ID - :query string block_id: block ID - - .. note:: - - Exactly one of the ``transaction_id`` or ``block_id`` query parameters must be - used together with this endpoint (see below for getting `transaction - statuses <#get--statuses?tx_id=tx_id>`_ and `block statuses - <#get--statuses?block_id=block_id>`_). - -.. _get_status_of_transaction: - -.. http:get:: /api/v1/statuses?transaction_id={transaction_id} - - Get the status of a transaction. - - The possible status values are ``undecided``, ``valid`` or ``backlog``. - If a transaction in neither of those states is found, a ``404 Not Found`` - HTTP status code is returned. `We're currently looking into ways to unambigously let the user know about a transaction's status that was included in an invalid block. `_ - - **Example request**: - - .. literalinclude:: http-samples/get-statuses-tx-request.http - :language: http - - **Example response**: - - .. literalinclude:: http-samples/get-statuses-tx-valid-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A transaction with that ID was found. - :statuscode 404: A transaction with that ID was not found. - - -.. http:get:: /api/v1/statuses?block_id={block_id} - - Get the status of a block. - - The possible status values are ``undecided``, ``valid`` or ``invalid``. - - **Example request**: - - .. literalinclude:: http-samples/get-statuses-block-request.http - :language: http - - **Example response**: - - .. literalinclude:: http-samples/get-statuses-block-valid-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A block with that ID was found. - :statuscode 404: A block with that ID was not found. - - Assets -------------------------------- @@ -674,37 +609,6 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``transaction_id``. -Votes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. http:get:: /api/v1/votes?block_id={block_id} - - Retrieve a list of votes for a certain block with ID ``block_id``. - To check for the validity of a vote, a user of this endpoint needs to - perform the `following steps: `_ - - 1. Check if the vote's ``node_pubkey`` is allowed to vote. - 2. Verify the vote's signature against the vote's body (``vote.vote``) and ``node_pubkey``. - - - :query string block_id: The block ID to filter the votes. - - **Example request**: - - .. literalinclude:: http-samples/get-vote-request.http - :language: http - - **Example response**: - - .. literalinclude:: http-samples/get-vote-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A list of votes voting for a block with ID ``block_id`` was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/votes``, without defining ``block_id``. - - .. _determining-the-api-root-url: Determining the API Root URL From 0139e53ab4af0981f66028491618571858fca695 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Feb 2018 14:09:08 +0100 Subject: [PATCH 21/28] rm code to gen example HTTP stuff for /statuses & /votes --- .../generate_http_server_api_documentation.py | 89 ------------------- 1 file changed, 89 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index bc12f913..62388e01 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -67,68 +67,6 @@ Content-Type: application/json """ -TPLS['post-tx-response'] = """\ -HTTP/1.1 202 Accepted -Content-Type: application/json - -%(tx)s -""" - - -TPLS['get-statuses-tx-request'] = """\ -GET /statuses?transaction_id=%(txid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-statuses-tx-invalid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "invalid" -} -""" - - -TPLS['get-statuses-tx-valid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "valid" -} -""" - - -TPLS['get-statuses-block-request'] = """\ -GET /api/v1/statuses?block_id=%(blockid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-statuses-block-invalid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "invalid" -} -""" - - -TPLS['get-statuses-block-valid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "valid" -} -""" - - TPLS['get-block-request'] = """\ GET /api/v1/blocks/%(blockid)s HTTP/1.1 Host: example.com @@ -159,21 +97,6 @@ Content-Type: application/json """ -TPLS['get-vote-request'] = """\ -GET /api/v1/votes?block_id=%(blockid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-vote-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -[%(vote)s] -""" - - def main(): """ Main function """ @@ -257,18 +180,6 @@ def main(): ctx['block_list'] = pretty_json(block_list) - # block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) - block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, - voters=[node_public], signature=signature) - ctx['block_transfer'] = pretty_json(block_transfer.to_dict()) - - # vote - vblock = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) - DUMMY_SHA3 = '0123456789abcdef' * 4 - b = Bigchain(public_key=node_public, private_key=node_private) - vote = b.vote(vblock.id, DUMMY_SHA3, True) - ctx['vote'] = pretty_json(vote) - base_path = os.path.join(os.path.dirname(__file__), 'source/http-samples') if not os.path.exists(base_path): From 7c3d7f4f6349395fbf5676551fe44a9352407d68 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Feb 2018 14:53:30 +0100 Subject: [PATCH 22/28] Tidy up some grammar and Tendermint refs --- docs/server/source/http-client-server-api.rst | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 76f1f4f1..6d6cc63a 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -130,11 +130,7 @@ Transactions `_ with three different modes to post transactions. By setting the mode, a new transaction can be pushed with a different mode than the default. The default mode is ``async``, which will return immediately and not wait to see if the transaction is valid. The ``sync`` mode will return after the transaction is validated, while ``commit`` - returns after the transaction is committed to a block. - - .. note:: - - This option is only available when using BigchainDB with Tendermint. + returns after the transaction is committed to a block. .. note:: @@ -160,8 +156,8 @@ Transactions .. note:: If the server is returning a ``202`` HTTP status code when ``mode=aysnc`` or ``mode=sync``, then the - transaction has been accepted for processing. The client can subscribe to - :ref:`WebSocket Event Stream API ` for listen for comitted transactions. + transaction has been accepted for processing. The client can subscribe to the + :ref:`WebSocket Event Stream API ` to listen for comitted transactions. :resheader Content-Type: ``application/json`` @@ -171,7 +167,8 @@ Transactions .. http:post:: /api/v1/transactions - This endpoint (without any parameters) will push a new transaction. If BigchainDB is used with Tendermint, the default mode ``async`` is used. + This endpoint (without any parameters) will push a new transaction. + Since no ``mode`` parameter is included, the default mode is assumed: ``async``. Transaction Outputs From 6982dfbe79fb945aa1a5ff83510fac92c07f10af Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Feb 2018 14:54:20 +0100 Subject: [PATCH 23/28] Put back the accidentally-removed post-tx-response --- docs/server/generate_http_server_api_documentation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 62388e01..27aa8205 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -67,6 +67,14 @@ Content-Type: application/json """ +TPLS['post-tx-response'] = """\ +HTTP/1.1 202 Accepted +Content-Type: application/json + +%(tx)s +""" + + TPLS['get-block-request'] = """\ GET /api/v1/blocks/%(blockid)s HTTP/1.1 Host: example.com From 7a292ba6ecda8c780ce0a23c54a2a5cc13e7427d Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Feb 2018 15:48:12 +0100 Subject: [PATCH 24/28] Edits to the common/schema/README.md file --- bigchaindb/common/schema/README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md index 76ff7f28..d43d1985 100644 --- a/bigchaindb/common/schema/README.md +++ b/bigchaindb/common/schema/README.md @@ -4,22 +4,24 @@ This directory contains the schemas for the different JSON documents BigchainDB The aim is to provide: -- a strict definition of the data structures used in BigchainDB -- a language independent tool to validate the structure of incoming/outcoming - data (there are several ready to use +- a strict definition of the data structures used in BigchainDB, +- a language-independent tool to validate the structure of incoming/outcoming + data. (There are several ready to use [implementations](http://json-schema.org/implementations.html) written in - different languages) + different languages.) ## Sources -The file defining the JSON Schema for votes (`vote.yaml`) is BigchainDB-specific. - The files defining the JSON Schema for transactions (`transaction_*.yaml`) -are copied from the [IPDB Protocol](https://github.com/ipdb/ipdb-protocol). -If you want to add a new version, you must add it to the IPDB Protocol first. -(You can't change existing versions. Those were used to validate old transactions +are based on the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). +If you want to add a new transaction version, +you must add it to the IPDB Transaction Spec first. +(You can't change the JSON Schema files for old versions. +Those were used to validate old transactions and are needed to re-check those transactions.) +The file defining the JSON Schema for votes (`vote.yaml`) is BigchainDB-specific. + ## Learn about JSON Schema A good resource is [Understanding JSON Schema](http://spacetelescope.github.io/understanding-json-schema/index.html). From 7f94242f85c1cb9240317571abda63dda75e8a90 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Feb 2018 16:24:44 +0100 Subject: [PATCH 25/28] Update all docs re/ votes when Tendermint integrated --- docs/root/source/decentralized.md | 2 +- docs/root/source/immutable.md | 2 +- docs/server/source/appendices/index.rst | 1 - docs/server/source/appendices/vote-yaml.rst | 20 --- docs/server/source/data-models/index.rst | 1 - docs/server/source/data-models/vote-model.rst | 120 ------------------ .../events/websocket-event-stream-api.rst | 8 +- docs/server/source/http-client-server-api.rst | 3 - 8 files changed, 4 insertions(+), 153 deletions(-) delete mode 100644 docs/server/source/appendices/vote-yaml.rst delete mode 100644 docs/server/source/data-models/vote-model.rst diff --git a/docs/root/source/decentralized.md b/docs/root/source/decentralized.md index 755a84a2..f38e12a2 100644 --- a/docs/root/source/decentralized.md +++ b/docs/root/source/decentralized.md @@ -4,7 +4,7 @@ Decentralization means that no one owns or controls everything, and there is no Ideally, each node in a BigchainDB cluster is owned and controlled by a different person or organization. Even if the cluster lives within one organization, it's still preferable to have each node controlled by a different person or subdivision. -We use the phrase "BigchainDB consortium" (or just "consortium") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A consortium requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each consortium, but it can be very decentralized (e.g. purely vote-based, where each node gets a vote, and there are no special leadership roles). +We use the phrase "BigchainDB consortium" (or just "consortium") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A consortium requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each consortium, but it can be very decentralized. If sharding is turned on (i.e. if the number of shards is larger than one), then the actual data is decentralized in that no one node stores all the data. diff --git a/docs/root/source/immutable.md b/docs/root/source/immutable.md index 55b61c27..8791c04e 100644 --- a/docs/root/source/immutable.md +++ b/docs/root/source/immutable.md @@ -13,7 +13,7 @@ Blockchain data can achieve immutability in several ways: 1. **External watchdogs.** A consortium may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For a consortium with publicly-readable data, the public can act as an auditor. 1. **Economic incentives.** Some blockchain systems make it very expensive to change old stored data. Examples include proof-of-work and proof-of-stake systems. BigchainDB doesn't use explicit incentives like those. 1. Data can be stored using fancy techniques, such as error-correction codes, to make some kinds of changes easier to undo. -1. **Cryptographic signatures** are often used as a way to check if messages (e.g. transactions, blocks or votes) have been tampered with enroute, and as a way to verify who signed the messages. In BigchainDB, each transaction must be signed (by one or more parties), each block is signed by the node that created it, and each vote is signed by the node that cast it. +1. **Cryptographic signatures** are often used as a way to check if messages (e.g. transactions) have been tampered with enroute, and as a way to verify who signed the messages. In BigchainDB, each transaction must be signed by one or more parties. 1. **Full or partial backups** may be recorded from time to time, possibly on magnetic tape storage, other blockchains, printouts, etc. 1. **Strong security.** Node owners can adopt and enforce strong security policies. 1. **Node diversity.** Diversity makes it so that no one thing (e.g. natural disaster or operating system bug) can compromise enough of the nodes. See [the section on the kinds of node diversity](diversity.html). diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 588f7ecd..db1695b5 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -18,4 +18,3 @@ Appendices licenses run-with-vagrant run-with-ansible - vote-yaml diff --git a/docs/server/source/appendices/vote-yaml.rst b/docs/server/source/appendices/vote-yaml.rst deleted file mode 100644 index 6613827e..00000000 --- a/docs/server/source/appendices/vote-yaml.rst +++ /dev/null @@ -1,20 +0,0 @@ -The Vote Schema File -==================== - -BigchainDB checks all :ref:`votes ` -(JSON documents) against a formal schema -defined in a JSON Schema file named vote.yaml. -The contents of that file are copied below. -To understand those contents -(i.e. JSON Schema), check out -`"Understanding JSON Schema" -`_ -by Michael Droettboom or -`json-schema.org `_. - - -vote.yaml ---------- - -.. literalinclude:: ../../../../bigchaindb/common/schema/vote.yaml - :language: yaml diff --git a/docs/server/source/data-models/index.rst b/docs/server/source/data-models/index.rst index 2827d219..94eb584e 100644 --- a/docs/server/source/data-models/index.rst +++ b/docs/server/source/data-models/index.rst @@ -9,4 +9,3 @@ Data Models inputs-outputs conditions block-model - vote-model diff --git a/docs/server/source/data-models/vote-model.rst b/docs/server/source/data-models/vote-model.rst deleted file mode 100644 index 1068dc0c..00000000 --- a/docs/server/source/data-models/vote-model.rst +++ /dev/null @@ -1,120 +0,0 @@ -The Vote Model -============== - -A vote is a JSON object with a particular schema, -as outlined in this page. -A vote must contain the following JSON keys -(also called names or fields): - -.. code-block:: json - - { - "node_pubkey": "", - "vote": { - "voting_for_block": "", - "previous_block": "", - "is_block_valid": "", - "invalid_reason": null, - "timestamp": "" - }, - "signature": "" - } - -.. note:: - - Votes have no ID (or ``"id"``), as far as users are concerned. - The backend database may use one internally, - but it's of no concern to users and it's never reported to them via APIs. - - -The JSON Keys in a Vote ------------------------ - -**node_pubkey** - -The public key of the node which cast this vote. -It's a string. -For more information about public keys, -see the `IPDB Transaction Spec `_ -section about cryptographic keys and signatures. - - -**vote.voting_for_block** - -The block ID that this vote is for. -It's a string. -For more information about block IDs, -see the page about :ref:`blocks `. - - -**vote.previous_block** - -The block ID of the block "before" the block that this vote is for, -according to the node which cast this vote. -It's a string. -(It's possible for different nodes to see different block orders.) -For more information about block IDs, -see the page about :ref:`blocks `. - - -**vote.is_block_valid** - -``true`` if the node which cast this vote considered the block in question to be valid, -and ``false`` otherwise. -Note that it's a *boolean* (i.e. ``true`` or ``false``), not a string. - - -**vote.invalid_reason** - -Always ``null``, that is, it's not being used. -It may be used or dropped in a future version. -See `bigchaindb/bigchaindb issue #217 -`_ on GitHub. - - -**vote.timestamp** - -The `Unix time `_ -when the vote was created, according to the node which created it. -It's a string representation of an integer. - - -**signature** - -The cryptographic signature of the inner ``vote`` -by the node that created the vote -(i.e. the node with public key ``node_pubkey``). -To compute that: - -#. Construct an :term:`associative array` ``d`` containing the contents of the inner ``vote`` - (i.e. ``vote.voting_for_block``, ``vote.previous_block``, ``vote.is_block_valid``, - ``vote.invalid_reason``, ``vote.timestamp``, and their values). -#. Compute ``signature = sig_of_aa(d, private_key)``, where ``private_key`` - is the node's private key (i.e. ``node_pubkey`` and ``private_key`` are a key pair). - There's pseudocode for the ``sig_of_aa()`` function - on the `IPDB Transaction Spec `_ - section about cryptographic keys and signatures. - -The Vote Schema ---------------- - -BigchainDB checks all votes (JSON documents) against a formal schema -defined in a :ref:`JSON Schema file named vote.yaml `. - - -An Example Vote ---------------- - -.. code-block:: json - - { - "node_pubkey": "3ZCsVWPAhPTqHx9wZVxp9Se54pcNeeM5mQvnozDWyDR9", - "vote": { - "voting_for_block": "11c3a3fcc9efa4fc4332a0849fc39b58e403ff37794a7d1fdfb9e7703a94a274", - "previous_block": "3dd1441018b782a50607dc4c7f83a0f0a23eb257f4b6a8d99330dfff41271e0d", - "is_block_valid": true, - "invalid_reason": null, - "timestamp": "1509977988" - }, - "signature": "3tW2EBVgxaZTE6nixVd9QEQf1vUxqPmQaNAMdCHc7zHik5KEosdkwScGYt4VhiHDTB6BCxTUzmqu3P7oP93tRWfj" - } diff --git a/docs/server/source/events/websocket-event-stream-api.rst b/docs/server/source/events/websocket-event-stream-api.rst index 7eb3f3f5..6603650b 100644 --- a/docs/server/source/events/websocket-event-stream-api.rst +++ b/docs/server/source/events/websocket-event-stream-api.rst @@ -69,7 +69,7 @@ All messages sent in a stream are in the JSON format. For simplicity, BigchainDB initially only provides a stream for all validated transactions. In the future, we may provide streams for other - information, such as new blocks, new votes, or invalid transactions. We may + information. We may also provide the ability to filter the stream for specific qualities, such as a specific ``output``'s ``public_key``. @@ -99,8 +99,4 @@ Example message: .. note:: Transactions in BigchainDB are validated in batches ("blocks") and will, - therefore, be streamed in batches. Each block can contain up to a 1000 - transactions, ordered by the time at which they were included in the block. - The ``/valid_transactions`` stream will send these transactions in the same - order that the block stored them in, but this does **NOT** guarantee that - you will recieve the events in that same order. + therefore, be streamed in batches. diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 6d6cc63a..b04d5bbd 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -518,9 +518,6 @@ a certain transaction with ``transaction_id`` occured in (a transaction can occu either gets rejected or validated by the system). This endpoint gives the ability to drill down on the lifecycle of a transaction -The `votes endpoint <#votes>`_ contains all the voting information for a specific block. So after retrieving the -``block_id`` for a given ``transaction_id``, one can now simply inspect the votes that happened at a specific time on that block. - Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 1b6d0adc01a6bb9a8df66083abf0cd65d4e41dbd Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 19 Feb 2018 12:04:10 +0100 Subject: [PATCH 26/28] removed code for all /statuses endpoints --- bigchaindb/web/routes.py | 2 - bigchaindb/web/views/info.py | 1 - bigchaindb/web/views/statuses.py | 45 --------------- tests/web/test_info.py | 2 - tests/web/test_statuses.py | 94 -------------------------------- tests/web/test_transactions.py | 3 - 6 files changed, 147 deletions(-) delete mode 100644 bigchaindb/web/views/statuses.py delete mode 100644 tests/web/test_statuses.py diff --git a/bigchaindb/web/routes.py b/bigchaindb/web/routes.py index b5fd8eb3..5b6185ba 100644 --- a/bigchaindb/web/routes.py +++ b/bigchaindb/web/routes.py @@ -5,7 +5,6 @@ from bigchaindb.web.views import ( metadata, blocks, info, - statuses, transactions as tx, outputs, votes, @@ -31,7 +30,6 @@ ROUTES_API_V1 = [ r('metadata/', metadata.MetadataApi), r('blocks/', blocks.BlockApi), r('blocks/', blocks.BlockListApi), - r('statuses/', statuses.StatusApi), r('transactions/', tx.TransactionApi), r('transactions', tx.TransactionListApi), r('outputs/', outputs.OutputListApi), diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index d6162a34..205f4930 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -46,7 +46,6 @@ def get_api_v1_info(api_prefix): return { 'docs': ''.join(docs_url), 'transactions': '{}transactions/'.format(api_prefix), - 'statuses': '{}statuses/'.format(api_prefix), 'assets': '{}assets/'.format(api_prefix), 'outputs': '{}outputs/'.format(api_prefix), 'streams': websocket_root, diff --git a/bigchaindb/web/views/statuses.py b/bigchaindb/web/views/statuses.py deleted file mode 100644 index 0044c334..00000000 --- a/bigchaindb/web/views/statuses.py +++ /dev/null @@ -1,45 +0,0 @@ -"""This module provides the blueprint for the statuses API endpoints. - -For more information please refer to the documentation: http://bigchaindb.com/http-api -""" -from flask import current_app -from flask_restful import Resource, reqparse - -from bigchaindb.web.views.base import make_error - - -class StatusApi(Resource): - def get(self): - """API endpoint to get details about the status of a transaction or a block. - - Return: - A ``dict`` in the format ``{'status': }``, where - ```` is one of "valid", "invalid", "undecided", "backlog". - """ - parser = reqparse.RequestParser() - parser.add_argument('transaction_id', type=str) - parser.add_argument('block_id', type=str) - - args = parser.parse_args(strict=True) - tx_id = args['transaction_id'] - block_id = args['block_id'] - - # logical xor - exactly one query argument required - if bool(tx_id) == bool(block_id): - return make_error(400, 'Provide exactly one query parameter. Choices are: block_id, transaction_id') - - pool = current_app.config['bigchain_pool'] - status = None - - with pool() as bigchain: - if tx_id: - status = bigchain.get_status(tx_id) - elif block_id: - _, status = bigchain.get_block(block_id=block_id, include_status=True) - - if not status: - return make_error(404) - - return { - 'status': status - } diff --git a/tests/web/test_info.py b/tests/web/test_info.py index 459f3e05..f682b5e3 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -13,7 +13,6 @@ def test_api_root_endpoint(client, wsserver_base_url): 'v1': { 'docs': ''.join(docs_url), 'transactions': '/api/v1/transactions/', - 'statuses': '/api/v1/statuses/', 'assets': '/api/v1/assets/', 'outputs': '/api/v1/outputs/', 'streams': '{}/api/v1/streams/valid_transactions'.format( @@ -37,7 +36,6 @@ def test_api_v1_endpoint(client, wsserver_base_url): api_v1_info = { 'docs': ''.join(docs_url), 'transactions': '/transactions/', - 'statuses': '/statuses/', 'assets': '/assets/', 'outputs': '/outputs/', 'streams': '{}/api/v1/streams/valid_transactions'.format( diff --git a/tests/web/test_statuses.py b/tests/web/test_statuses.py deleted file mode 100644 index 7c2a7c14..00000000 --- a/tests/web/test_statuses.py +++ /dev/null @@ -1,94 +0,0 @@ -import pytest - -from bigchaindb.models import Transaction - -STATUSES_ENDPOINT = '/api/v1/statuses' - - -@pytest.mark.bdb -@pytest.mark.usefixtures('inputs') -def test_get_transaction_status_endpoint(b, client, user_pk): - input_tx = b.get_owned_ids(user_pk).pop() - tx, status = b.get_transaction(input_tx.txid, include_status=True) - res = client.get(STATUSES_ENDPOINT + '?transaction_id=' + input_tx.txid) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -def test_get_transaction_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + '?transaction_id=123') - assert res.status_code == 404 - - -@pytest.mark.bdb -def test_get_block_status_endpoint_undecided(b, client): - tx = Transaction.create([b.me], [([b.me], 1)]) - tx = tx.sign([b.me_private]) - - block = b.create_block([tx]) - b.write_block(block) - - status = b.block_election_status(block) - - res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -@pytest.mark.usefixtures('inputs') -def test_get_block_status_endpoint_valid(b, client): - tx = Transaction.create([b.me], [([b.me], 1)]) - tx = tx.sign([b.me_private]) - - block = b.create_block([tx]) - b.write_block(block) - - # vote the block valid - vote = b.vote(block.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - status = b.block_election_status(block) - - res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -@pytest.mark.usefixtures('inputs') -def test_get_block_status_endpoint_invalid(b, client): - tx = Transaction.create([b.me], [([b.me], 1)]) - tx = tx.sign([b.me_private]) - - block = b.create_block([tx]) - b.write_block(block) - - # vote the block valid - vote = b.vote(block.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - - status = b.block_election_status(block) - - res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -def test_get_block_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + '?block_id=123') - assert res.status_code == 404 - - -@pytest.mark.bdb -def test_get_status_endpoint_returns_400_bad_query_params(client): - res = client.get(STATUSES_ENDPOINT) - assert res.status_code == 400 - - res = client.get(STATUSES_ENDPOINT + '?ts_id=123') - assert res.status_code == 400 - - res = client.get(STATUSES_ENDPOINT + '?transaction_id=123&block_id=123') - assert res.status_code == 400 diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index a67d790e..f376bf09 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -44,9 +44,6 @@ def test_post_create_transaction_endpoint(b, client): assert res.status_code == 202 - assert '../statuses?transaction_id={}'.format(tx.id) in \ - res.headers['Location'] - assert res.json['inputs'][0]['owners_before'][0] == user_pub assert res.json['outputs'][0]['public_keys'][0] == user_pub From c5fdaf24ad12cf9b7af83adc0244de217b5a8f20 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Feb 2018 12:58:23 +0100 Subject: [PATCH 27/28] Fix the requirements.txt file for docs/server, update README.md --- docs/README.md | 8 ++++++++ docs/server/requirements.txt | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 220569e2..4071546f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,14 @@ If you want to generate the HTML version of the long-form documentation on your pip install -r requirements.txt ``` +If you're building the *Server* docs (in `docs/server`) then you must also do: +```bash +pip install -e ../../ +``` + +Note: Don't put `-e ../../` in the `requirements.txt` file. That will work locally +but not on ReadTheDocs. + You can then generate the HTML documentation _in that subdirectory_ by doing: ```bash make html diff --git a/docs/server/requirements.txt b/docs/server/requirements.txt index cd06eab9..45600057 100644 --- a/docs/server/requirements.txt +++ b/docs/server/requirements.txt @@ -4,4 +4,3 @@ sphinx-rtd-theme>=0.1.9 sphinxcontrib-napoleon>=0.4.4 sphinxcontrib-httpdomain>=1.5.0 pyyaml>=3.12 -bigchaindb From 5bfa8e29d8aa689704f03888e83f118404f4c755 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Wed, 21 Feb 2018 15:20:12 +0530 Subject: [PATCH 28/28] Crash recovery mechanism (#2045) * Crash recovery mechanism * Propogate exception * Added docs and crash receovery during block write * Fix flake8 issue * Remove approach 1 for crash recovery, recover db on 'bigchiandb start' * Fix CI build issues * Remove documentation --- bigchaindb/backend/localmongodb/query.py | 59 ++++++++---- bigchaindb/backend/query.py | 28 ++++++ bigchaindb/commands/bigchaindb.py | 17 ++++ bigchaindb/tendermint/lib.py | 4 + compose/travis/Dockerfile | 1 + docker-compose.travis.yml | 5 +- tests/backend/localmongodb/test_queries.py | 42 +++++++++ tests/commands/test_commands.py | 101 ++++++++++++++++++++- tests/conftest.py | 4 +- tmdata/genesis.json | 1 + tmdata/priv_validator.json | 1 + 11 files changed, 240 insertions(+), 23 deletions(-) create mode 100644 tmdata/genesis.json create mode 100644 tmdata/priv_validator.json diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 8e4f4c81..3a21c802 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -24,11 +24,8 @@ def store_transaction(conn, signed_transaction): @register_query(LocalMongoDBConnection) def store_transactions(conn, signed_transactions): - try: - return conn.run(conn.collection('transactions') - .insert_many(signed_transactions)) - except DuplicateKeyError: - pass + return conn.run(conn.collection('transactions') + .insert_many(signed_transactions)) @register_query(LocalMongoDBConnection) @@ -54,12 +51,9 @@ def get_transactions(conn, transaction_ids): @register_query(LocalMongoDBConnection) def store_metadatas(conn, metadata): - try: - return conn.run( - conn.collection('metadata') - .insert_many(metadata, ordered=False)) - except DuplicateKeyError: - pass + return conn.run( + conn.collection('metadata') + .insert_many(metadata, ordered=False)) @register_query(LocalMongoDBConnection) @@ -82,12 +76,9 @@ def store_asset(conn, asset): @register_query(LocalMongoDBConnection) def store_assets(conn, assets): - try: - return conn.run( - conn.collection('assets') - .insert_many(assets, ordered=False)) - except DuplicateKeyError: - pass + return conn.run( + conn.collection('assets') + .insert_many(assets, ordered=False)) @register_query(LocalMongoDBConnection) @@ -201,6 +192,40 @@ def get_block_with_transaction(conn, txid): projection={'_id': False, 'height': True})) +@register_query(LocalMongoDBConnection) +def delete_zombie_transactions(conn): + txns = conn.run(conn.collection('transactions').find({})) + for txn in txns: + txn_id = txn['id'] + block = list(get_block_with_transaction(conn, txn_id)) + if len(block) == 0: + delete_transaction(conn, txn_id) + + +def delete_transaction(conn, txn_id): + conn.run( + conn.collection('transactions').delete_one({'id': txn_id})) + conn.run( + conn.collection('assets').delete_one({'id': txn_id})) + conn.run( + conn.collection('metadata').delete_one({'id': txn_id})) + + +@register_query(LocalMongoDBConnection) +def delete_latest_block(conn): + block = get_latest_block(conn) + txn_ids = block['transactions'] + delete_transactions(conn, txn_ids) + conn.run(conn.collection('blocks').delete_one({'height': block['height']})) + + +@register_query(LocalMongoDBConnection) +def delete_transactions(conn, txn_ids): + conn.run(conn.collection('assets').delete_many({'id': {'$in': txn_ids}})) + conn.run(conn.collection('metadata').delete_many({'id': {'$in': txn_ids}})) + conn.run(conn.collection('transactions').delete_many({'id': {'$in': txn_ids}})) + + @register_query(LocalMongoDBConnection) def store_unspent_outputs(conn, *unspent_outputs): try: diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index e2147e98..1e59b573 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -549,6 +549,13 @@ def store_block(conn, block): raise NotImplementedError +@singledispatch +def delete_zombie_transactions(conn): + """Delete transactions not included in any block""" + + raise NotImplementedError + + @singledispatch def store_unspent_outputs(connection, unspent_outputs): """Store unspent outputs in ``utxo_set`` table.""" @@ -556,6 +563,13 @@ def store_unspent_outputs(connection, unspent_outputs): raise NotImplementedError +@singledispatch +def delete_latest_block(conn): + """Delete the latest block along with its transactions""" + + raise NotImplementedError + + @singledispatch def delete_unspent_outputs(connection, unspent_outputs): """Delete unspent outputs in ``utxo_set`` table.""" @@ -563,6 +577,20 @@ def delete_unspent_outputs(connection, unspent_outputs): raise NotImplementedError +@singledispatch +def delete_transactions(conn, txn_ids): + """Delete transactions from database + + Args: + txn_ids (list): list of transaction ids + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + @singledispatch def get_unspent_outputs(connection, *, query=None): """Retrieves unspent outputs. diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 3b2edb5a..a866fcaa 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -15,8 +15,10 @@ from bigchaindb.common.exceptions import (StartupError, KeypairNotFoundException, DatabaseDoesNotExist) import bigchaindb +from bigchaindb.tendermint.core import BigchainDB from bigchaindb import backend from bigchaindb.backend import schema +from bigchaindb.backend import query from bigchaindb.backend.admin import (set_replicas, set_shards, add_replicas, remove_replicas) from bigchaindb.backend.exceptions import OperationError @@ -154,6 +156,19 @@ def run_init(args): print('If you wish to re-initialize it, first drop it.', file=sys.stderr) +def run_recover(b): + query.delete_zombie_transactions(b.connection) + + tendermint_height = b.get_latest_block_height_from_tendermint() + block = b.get_latest_block() + + if block: + while block['height'] > tendermint_height: + logger.info('BigchainDB is ahead of tendermint, removing block %s', block['height']) + query.delete_latest_block(b.connection) + block = b.get_latest_block() + + @configure_bigchaindb def run_drop(args): """Drop the database""" @@ -178,6 +193,8 @@ def run_start(args): """Start the processes to run the node""" logger.info('BigchainDB Version %s', bigchaindb.__version__) + run_recover(BigchainDB()) + if args.allow_temp_keypair: if not (bigchaindb.config['keypair']['private'] or bigchaindb.config['keypair']['public']): diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index d4e6647f..88f65ddd 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -50,6 +50,10 @@ class BigchainDB(Bigchain): """Submit a valid transaction to the mempool.""" self.post_transaction(transaction, mode) + def get_latest_block_height_from_tendermint(self): + r = requests.get(ENDPOINT + 'status') + return r.json()['result']['latest_block_height'] + def store_transaction(self, transaction): """Store a valid transaction to the transactions collection.""" diff --git a/compose/travis/Dockerfile b/compose/travis/Dockerfile index 7a634684..cb37fa07 100644 --- a/compose/travis/Dockerfile +++ b/compose/travis/Dockerfile @@ -25,5 +25,6 @@ ENV BIGCHAINDB_TENDERMINT_PORT 46657 RUN mkdir -p /usr/src/app COPY . /usr/src/app/ WORKDIR /usr/src/app +RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf RUN pip install --no-cache-dir .[test] RUN bigchaindb -y configure "$backend" diff --git a/docker-compose.travis.yml b/docker-compose.travis.yml index e8d1c583..7707a276 100644 --- a/docker-compose.travis.yml +++ b/docker-compose.travis.yml @@ -29,5 +29,6 @@ services: image: tendermint/tendermint:0.13 volumes: - ./tmdata/config.toml:/tendermint/config.toml - entrypoint: '' - command: bash -c "tendermint init && tendermint node" + - ./tmdata/genesis.json:/tendermint/genesis.json + - ./tmdata/priv_validator.json:/tendermint/priv_validator.json + entrypoint: ["/bin/tendermint", "node", "--proxy_app=dummy"] diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index 768bfbf9..ec483793 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -199,6 +199,48 @@ def test_get_block(): assert block['height'] == 3 +def test_delete_zombie_transactions(signed_create_tx, signed_transfer_tx): + from bigchaindb.backend import connect, query + from bigchaindb.tendermint.lib import Block + conn = connect() + + conn.db.transactions.insert_one(signed_create_tx.to_dict()) + query.store_asset(conn, {'id': signed_create_tx.id}) + block = Block(app_hash='random_utxo', + height=3, + transactions=[signed_create_tx.id]) + query.store_block(conn, block._asdict()) + + conn.db.transactions.insert_one(signed_transfer_tx.to_dict()) + query.store_metadatas(conn, [{'id': signed_transfer_tx.id}]) + + query.delete_zombie_transactions(conn) + assert query.get_transaction(conn, signed_transfer_tx.id) is None + assert query.get_asset(conn, signed_transfer_tx.id) is None + assert list(query.get_metadata(conn, [signed_transfer_tx.id])) == [] + + assert query.get_transaction(conn, signed_create_tx.id) is not None + assert query.get_asset(conn, signed_create_tx.id) is not None + + +def test_delete_latest_block(signed_create_tx, signed_transfer_tx): + from bigchaindb.backend import connect, query + from bigchaindb.tendermint.lib import Block + conn = connect() + + conn.db.transactions.insert_one(signed_create_tx.to_dict()) + query.store_asset(conn, {'id': signed_create_tx.id}) + block = Block(app_hash='random_utxo', + height=51, + transactions=[signed_create_tx.id]) + query.store_block(conn, block._asdict()) + query.delete_latest_block(conn) + + assert query.get_transaction(conn, signed_create_tx.id) is None + assert query.get_asset(conn, signed_create_tx.id) is None + assert query.get_block(conn, 51) is None + + def test_delete_unspent_outputs(db_context, utxoset): from bigchaindb.backend import query unspent_outputs, utxo_collection = utxoset diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 423e4614..aa0df51e 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -6,14 +6,15 @@ import copy import pytest +@pytest.mark.tendermint def test_make_sure_we_dont_remove_any_command(): # thanks to: http://stackoverflow.com/a/18161115/597097 from bigchaindb.commands.bigchaindb import create_parser parser = create_parser() - assert parser.parse_args(['configure', 'rethinkdb']).command - assert parser.parse_args(['configure', 'mongodb']).command + assert parser.parse_args(['configure', 'localmongodb']).command + assert parser.parse_args(['configure', 'localmongodb']).command assert parser.parse_args(['show-config']).command assert parser.parse_args(['export-my-pubkey']).command assert parser.parse_args(['init']).command @@ -517,3 +518,99 @@ def test_run_remove_replicas(mock_remove_replicas): assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 mock_remove_replicas.reset_mock() + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_recover_db_from_zombie_txn(b, monkeypatch): + from bigchaindb.commands.bigchaindb import run_recover + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.tendermint.lib import Block + from bigchaindb import backend + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset={'cycle': 'hero'}, + metadata={'name': 'hohenheim'}) \ + .sign([alice.private_key]) + b.store_bulk_transactions([tx]) + block = Block(app_hash='random_app_hash', height=10, + transactions=[])._asdict() + b.store_block(block) + + def mock_get(uri): + return MockResponse(10) + monkeypatch.setattr('requests.get', mock_get) + + run_recover(b) + + assert list(backend.query.get_metadata(b.connection, [tx.id])) == [] + assert not backend.query.get_asset(b.connection, tx.id) + assert not b.get_transaction(tx.id) + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_recover_db_from_zombie_block(b, monkeypatch): + from bigchaindb.commands.bigchaindb import run_recover + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.tendermint.lib import Block + from bigchaindb import backend + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset={'cycle': 'hero'}, + metadata={'name': 'hohenheim'}) \ + .sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + block9 = Block(app_hash='random_app_hash', height=9, + transactions=[])._asdict() + b.store_block(block9) + block10 = Block(app_hash='random_app_hash', height=10, + transactions=[tx.id])._asdict() + b.store_block(block10) + + def mock_get(uri): + return MockResponse(9) + monkeypatch.setattr('requests.get', mock_get) + + run_recover(b) + + assert list(backend.query.get_metadata(b.connection, [tx.id])) == [] + assert not backend.query.get_asset(b.connection, tx.id) + assert not b.get_transaction(tx.id) + + block = b.get_latest_block() + assert block['height'] == 9 + + +@pytest.mark.tendermint +@patch('bigchaindb.config_utils.autoconfigure') +@patch('bigchaindb.commands.bigchaindb.run_recover') +@patch('bigchaindb.tendermint.commands.start') +def test_recover_db_on_start(mock_autoconfigure, + mock_run_recover, + mock_start, + mocked_setup_logging): + from bigchaindb.commands.bigchaindb import run_start + args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True, + skip_initialize_database=False) + run_start(args) + + assert mock_run_recover.called + assert mock_start.called + + +# Helper +class MockResponse(): + + def __init__(self, height): + self.height = height + + def json(self): + return {'result': {'latest_block_height': self.height}} diff --git a/tests/conftest.py b/tests/conftest.py index 4828d5d6..3899e71d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -328,8 +328,8 @@ def merlin_pubkey(merlin): @pytest.fixture def b(): - from bigchaindb import Bigchain - return Bigchain() + from bigchaindb.tendermint import BigchainDB + return BigchainDB() @pytest.fixture diff --git a/tmdata/genesis.json b/tmdata/genesis.json new file mode 100644 index 00000000..877500df --- /dev/null +++ b/tmdata/genesis.json @@ -0,0 +1 @@ +{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-JCYeEN","validators":[{"pub_key":{"type":"ed25519","data":"0C988282C02CFF72E5E296DB78CE26D922178549B327C375D992548C9AFCCE6D"},"power":10,"name":""}],"app_hash":""} diff --git a/tmdata/priv_validator.json b/tmdata/priv_validator.json new file mode 100644 index 00000000..1dc74a58 --- /dev/null +++ b/tmdata/priv_validator.json @@ -0,0 +1 @@ +{"address":"E6CB05DA326F70BB4CC0A4AF83FC3BBF70B9A4D5","pub_key":{"type":"ed25519","data":"0C988282C02CFF72E5E296DB78CE26D922178549B327C375D992548C9AFCCE6D"},"last_height":0,"last_round":0,"last_step":0,"last_signature":null,"priv_key":{"type":"ed25519","data":"D4488996BDF92CE1D80670C66923D4996AE1B772FE0F76DAE33EDC410DC1D58F0C988282C02CFF72E5E296DB78CE26D922178549B327C375D992548C9AFCCE6D"}}