From b122e7e83b17e919cbc2422afe2c57af996e866d Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 16 Jul 2018 15:39:07 +0200 Subject: [PATCH 01/11] Last PR before the BigchainDB 2.0 beta 2 release (#2382) * Updated CHANGELOG.md for the 2.0b2 release * Update bigchaindb docker image names for k8s for the 2.0b2 release * Update version.py for the 2.0b2 release * In CHANGELOG.md, added note re port 46657->26657 --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++ bigchaindb/version.py | 4 +-- k8s/bigchaindb/bigchaindb-ss.yaml | 2 +- k8s/dev-setup/bigchaindb.yaml | 2 +- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 108468fb..bffdd155 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,51 @@ For reference, the possible headings are: * **Known Issues** * **Notes** +## [2.0 Beta 2] - 2018-07-16 + +Tag name: v2.0.0b2 + +### Added + +* Added new configuration settings `tendermint.host` and `tendermint.port`. [Pull request #2342](https://github.com/bigchaindb/bigchaindb/pull/2342) +* Added tests to ensure that BigchainDB gracefully handles "nasty" strings in keys and values. [Pull request #2334](https://github.com/bigchaindb/bigchaindb/pull/2334) +* Added a new logging handler to capture benchmark stats to a separate file. [Pull request #2349](https://github.com/bigchaindb/bigchaindb/pull/2349) + +### Changed + +* Changed the names of BigchainDB processes (Python processes) to include 'bigchaindb', so they are easier to spot and find. [Pull request #2354](https://github.com/bigchaindb/bigchaindb/pull/2354) +* Updated all code to support the latest version of Tendermint. Note that the BigchainDB ABCI server now listens to port 26657 instead of 46657. Pull requests [#2375](https://github.com/bigchaindb/bigchaindb/pull/2375) and [#2380](https://github.com/bigchaindb/bigchaindb/pull/2380) + +### Removed + +Removed all support and code for the old backlog_reassign_delay setting. [Pull request #2332](https://github.com/bigchaindb/bigchaindb/pull/2332) + +### Fixed + +* Fixed a bug that sometimes arose when using Docker Compose. (Tendermint would freeze.) [Pull request #2341](https://github.com/bigchaindb/bigchaindb/pull/2341) +* Fixed a bug in the code that creates a MongoDB index for the "id" in the transactions collection. It works now, and performance is improved. [Pull request #2378](https://github.com/bigchaindb/bigchaindb/pull/2378) +* The logging server would keep runnning in some tear-down scenarios. It doesn't do that any more. [Pull request #2304](https://github.com/bigchaindb/bigchaindb/pull/2304) + +### External Contributors + +@hrntknr - [Pull request #2331](https://github.com/bigchaindb/bigchaindb/pull/2331) + +### Known Issues + +The `bigchaindb upsert-validator` subcommand is not working yet, but a solution ([BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21)) has been finalized and will be implemented before we release the final BigchainDB 2.0. + +### Notes + +* A lot of old/dead code was deleted. Pull requests +[#2319](https://github.com/bigchaindb/bigchaindb/pull/2319), +[#2338](https://github.com/bigchaindb/bigchaindb/pull/2338), +[#2357](https://github.com/bigchaindb/bigchaindb/pull/2357), +[#2365](https://github.com/bigchaindb/bigchaindb/pull/2365), +[#2366](https://github.com/bigchaindb/bigchaindb/pull/2366), +[#2368](https://github.com/bigchaindb/bigchaindb/pull/2368) and +[#2374](https://github.com/bigchaindb/bigchaindb/pull/2374) +* Improved the documentation page "How to setup a BigchainDB Network". [Pull Request #2312](https://github.com/bigchaindb/bigchaindb/pull/2312) + ## [2.0 Beta 1] - 2018-06-01 Tag name: v2.0.0b1 diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 174d8004..edc112cd 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '2.0.0b1' -__short_version__ = '2.0b1' +__version__ = '2.0.0b2' +__short_version__ = '2.0b2' diff --git a/k8s/bigchaindb/bigchaindb-ss.yaml b/k8s/bigchaindb/bigchaindb-ss.yaml index a61e037a..636da377 100644 --- a/k8s/bigchaindb/bigchaindb-ss.yaml +++ b/k8s/bigchaindb/bigchaindb-ss.yaml @@ -154,7 +154,7 @@ spec: timeoutSeconds: 15 # BigchainDB container - name: bigchaindb - image: bigchaindb/bigchaindb:2.0.0-beta1 + image: bigchaindb/bigchaindb:2.0.0-beta2 imagePullPolicy: Always args: - start diff --git a/k8s/dev-setup/bigchaindb.yaml b/k8s/dev-setup/bigchaindb.yaml index c9e6442c..e8db3042 100644 --- a/k8s/dev-setup/bigchaindb.yaml +++ b/k8s/dev-setup/bigchaindb.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:2.0.0-beta1 + image: bigchaindb/bigchaindb:2.0.0-beta2 imagePullPolicy: Always args: - start From fdf02835082399d8ab58a69fa992774f6a33e2e9 Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Mon, 16 Jul 2018 19:21:27 +0200 Subject: [PATCH 02/11] Problem: consensus and blocks are deprecated (#2384) Solution: remove test files for consensus and blocks --- tests/test_block_model.py | 292 -------------------------------------- tests/test_consensus.py | 0 2 files changed, 292 deletions(-) delete mode 100644 tests/test_block_model.py delete mode 100644 tests/test_consensus.py diff --git a/tests/test_block_model.py b/tests/test_block_model.py deleted file mode 100644 index dcc6e878..00000000 --- a/tests/test_block_model.py +++ /dev/null @@ -1,292 +0,0 @@ -import pytest -from pytest import raises - - -class TestBlockModel(object): - def test_block_initialization(self, monkeypatch): - from bigchaindb.models import Block - - monkeypatch.setattr('time.time', lambda: 1) - - block = Block() - assert block.transactions == [] - assert block.timestamp == '1' - assert block.node_pubkey is None - assert block.signature is None - - with raises(TypeError): - Block('not a list or None') - - def test_block_serialization(self, b, alice): - from bigchaindb.common.crypto import hash_data - from bigchaindb.common.utils import gen_timestamp, serialize - from bigchaindb.models import Block, Transaction - - transactions = [Transaction.create([alice.public_key], [([alice.public_key], 1)])] - timestamp = gen_timestamp() - expected_block = { - 'timestamp': timestamp, - 'transactions': [tx.to_dict() for tx in transactions], - 'node_pubkey': alice.public_key, - } - expected = { - 'id': hash_data(serialize(expected_block)), - 'block': expected_block, - 'signature': None, - } - - block = Block(transactions, alice.public_key, timestamp) - - assert block.to_dict() == expected - - def test_block_invalid_serializaton(self): - from bigchaindb.models import Block - - block = Block([]) - with raises(ValueError): - block.to_dict() - - def test_block_deserialization(self, b, alice): - from bigchaindb.common.crypto import hash_data - from bigchaindb.common.utils import gen_timestamp, serialize - from bigchaindb.models import Block, Transaction - - transaction = Transaction.create([alice.public_key], [([alice.public_key], 1)]) - transaction.sign([alice.private_key]) - timestamp = gen_timestamp() - expected = Block([transaction], alice.public_key, timestamp) - - block = { - 'timestamp': timestamp, - 'transactions': [transaction.to_dict()], - 'node_pubkey': alice.public_key, - } - - block_body = { - 'id': hash_data(serialize(block)), - 'block': block, - 'signature': None, - } - - assert expected == Block.from_dict(block_body) - - def test_block_invalid_id_deserialization(self, b, alice): - from bigchaindb.common.exceptions import InvalidHash - from bigchaindb.models import Block - - block = { - 'id': 'an invalid id', - 'block': { - 'node_pubkey': alice.public_key, - } - } - - with raises(InvalidHash): - Block.from_dict(block) - - def test_block_invalid_signature(self, b, alice): - from bigchaindb.common.crypto import hash_data - from bigchaindb.common.exceptions import InvalidSignature - from bigchaindb.common.utils import gen_timestamp, serialize - from bigchaindb.models import Block, Transaction - - transaction = Transaction.create([alice.public_key], [([alice.public_key], 1)]) - transaction.sign([alice.private_key]) - timestamp = gen_timestamp() - - block = { - 'timestamp': timestamp, - 'transactions': [transaction.to_dict()], - 'node_pubkey': alice.public_key, - } - - block_body = { - 'id': hash_data(serialize(block)), - 'block': block, - 'signature': 'an invalid signature', - } - - with raises(InvalidSignature): - Block.from_dict(block_body).validate(b) - - def test_compare_blocks(self, b, alice): - from bigchaindb.models import Block, Transaction - - transactions = [Transaction.create([alice.public_key], [([alice.public_key], 1)])] - - assert Block() != 'invalid comparison' - assert Block(transactions) == Block(transactions) - - def test_sign_block(self, b, alice): - from bigchaindb.common.crypto import PrivateKey, PublicKey - from bigchaindb.common.utils import gen_timestamp, serialize - from bigchaindb.models import Block, Transaction - - transactions = [Transaction.create([alice.public_key], [([alice.public_key], 1)])] - timestamp = gen_timestamp() - expected_block = { - 'timestamp': timestamp, - 'transactions': [tx.to_dict() for tx in transactions], - 'node_pubkey': alice.public_key, - } - expected_block_serialized = serialize(expected_block).encode() - expected = PrivateKey(alice.private_key).sign(expected_block_serialized) - block = Block(transactions, alice.public_key, timestamp) - block = block.sign(alice.private_key) - assert block.signature == expected.decode() - - public_key = PublicKey(alice.public_key) - assert public_key.verify(expected_block_serialized, block.signature) - - def test_block_dupe_tx(self, b, alice): - from bigchaindb.models import Block, Transaction - from bigchaindb.common.exceptions import DuplicateTransaction - - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) - block = Block([tx, tx], alice.public_key) - block.sign(alice.private_key) - b.store_block(block.to_dict()) - with raises(DuplicateTransaction): - block.validate(b) - - def test_decouple_assets(self, b, alice): - from bigchaindb.models import Block, Transaction - - assets = [ - {'msg': '1'}, - {'msg': '2'}, - {'msg': '3'}, - ] - - txs = [] - # create 3 assets - for asset in assets: - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset=asset) - tx.sign([alice.private_key]) - txs.append(tx) - - # create a `TRANSFER` transaction. - # the asset in `TRANSFER` transactions is not extracted - tx = Transaction.transfer(txs[0].to_inputs(), [([alice.public_key], 1)], - asset_id=txs[0].id) - tx.sign([alice.private_key]) - txs.append(tx) - - # create the block - block = Block(txs) - # decouple assets - assets_from_block, block_dict = block.decouple_assets() - - assert len(assets_from_block) == 3 - for i in range(3): - assert assets_from_block[i]['data'] == assets[i] - assert assets_from_block[i]['id'] == txs[i].id - - # check the `TRANSFER` transaction was not changed - assert block.transactions[3].to_dict() == \ - block_dict['block']['transactions'][3] - - def test_couple_assets(self, b, alice): - from bigchaindb.models import Block, Transaction - - assets = [ - {'msg': '1'}, - {'msg': '2'}, - {'msg': '3'}, - ] - - txs = [] - # create 3 assets - for asset in assets: - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset=asset) - tx.sign([alice.private_key]) - txs.append(tx) - - # create a `TRANSFER` transaction. - # the asset in `TRANSFER` transactions is not extracted - tx = Transaction.transfer(txs[0].to_inputs(), [([alice.public_key], 1)], - asset_id=txs[0].id) - tx.sign([alice.private_key]) - txs.append(tx) - - # create the block - block = Block(txs) - # decouple assets - assets_from_block, block_dict = block.decouple_assets() - - # reconstruct the block - block_dict_reconstructed = Block.couple_assets(block_dict, - assets_from_block) - - # check that the reconstructed block is the same as the original block - assert block == Block.from_dict(block_dict_reconstructed) - - def test_get_asset_ids(self, b, alice): - from bigchaindb.models import Block, Transaction - - assets = [ - {'msg': '1'}, - {'msg': '2'}, - {'msg': '3'}, - ] - - txs = [] - # create 3 assets - for asset in assets: - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset=asset) - tx.sign([alice.private_key]) - txs.append(tx) - - # create a `TRANSFER` transaction. - # the asset in `TRANSFER` transactions is not extracted - tx = Transaction.transfer(txs[0].to_inputs(), [([alice.public_key], 1)], - asset_id=txs[0].id) - tx.sign([alice.private_key]) - txs.append(tx) - - # create the block - block = Block(txs) - # decouple assets - assets_from_block, block_dict = block.decouple_assets() - - # get the asset_ids and check that they are the same as the `CREATE` - # transactions - asset_ids = Block.get_asset_ids(block_dict) - assert asset_ids == [tx.id for tx in txs[:-1]] - - @pytest.mark.bdb - def test_from_db(self, b, alice): - from bigchaindb.models import Block, Transaction - - assets = [ - {'msg': '1'}, - {'msg': '2'}, - {'msg': '3'}, - ] - - txs = [] - # create 3 assets - for asset in assets: - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset=asset) - tx.sign([alice.private_key]) - txs.append(tx) - - # create a `TRANSFER` transaction. - # the asset in `TRANSFER` transactions is not extracted - tx = Transaction.transfer(txs[0].to_inputs(), [([alice.public_key], 1)], - asset_id=txs[0].id) - tx.sign([alice.private_key]) - txs.append(tx) - - # create the block - block = Block(txs) - # decouple assets - assets_from_block, block_dict = block.decouple_assets() - - # write the assets and block separately - b.write_assets(assets_from_block) - b.write_block(block) - - # check the reconstructed block is the same as the original block - block_from_db = Block.from_db(b, block_dict) - assert block == block_from_db diff --git a/tests/test_consensus.py b/tests/test_consensus.py deleted file mode 100644 index e69de29b..00000000 From d521a00925819fb560fd0a0e39d98f5c521d6ed9 Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Wed, 18 Jul 2018 10:52:33 +0200 Subject: [PATCH 03/11] Problem: a valid transaction is detected as double spend (#2389) Solution: query the wanted data per input --- bigchaindb/backend/localmongodb/query.py | 8 +++++--- tests/test_core.py | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 6aca2197..b2b381c4 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -99,11 +99,13 @@ def get_assets(conn, asset_ids): @register_query(LocalMongoDBConnection) def get_spent(conn, transaction_id, output): + query = {'inputs.fulfills': { + 'transaction_id': transaction_id, + 'output_index': output}} + return conn.run( conn.collection('transactions') - .find({'inputs.fulfills.transaction_id': transaction_id, - 'inputs.fulfills.output_index': output}, - {'_id': 0})) + .find(query, {'_id': 0})) @register_query(LocalMongoDBConnection) diff --git a/tests/test_core.py b/tests/test_core.py index 9f6150e6..5f89baa1 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,6 +1,10 @@ import pytest +pytestmark = pytest.mark.tendermint + + +@pytest.mark.skipif(reason='will be fixed in another PR') @pytest.fixture def config(request, monkeypatch): backend = request.config.getoption('--database-backend') @@ -25,6 +29,7 @@ def config(request, monkeypatch): return config +@pytest.mark.skipif(reason='will be fixed in another PR') def test_bigchain_class_default_initialization(config): from bigchaindb.tendermint import BigchainDB from bigchaindb.consensus import BaseConsensusRules @@ -37,6 +42,7 @@ def test_bigchain_class_default_initialization(config): assert bigchain.consensus == BaseConsensusRules +@pytest.mark.skipif(reason='will be fixed in another PR') def test_bigchain_class_initialization_with_parameters(config): from bigchaindb.tendermint import BigchainDB from bigchaindb.backend import connect @@ -56,6 +62,7 @@ def test_bigchain_class_initialization_with_parameters(config): assert bigchain.consensus == BaseConsensusRules +@pytest.mark.skipif(reason='will be fixed in another PR') def test_get_blocks_status_containing_tx(monkeypatch): from bigchaindb.backend import query as backend_query from bigchaindb.tendermint import BigchainDB @@ -77,6 +84,8 @@ def test_get_spent_issue_1271(b, alice, bob, carol): [carol.public_key], [([carol.public_key], 8)], ).sign([carol.private_key]) + assert b.validate_transaction(tx_1) + b.store_bulk_transactions([tx_1]) tx_2 = Transaction.transfer( tx_1.to_inputs(), @@ -85,6 +94,8 @@ def test_get_spent_issue_1271(b, alice, bob, carol): ([carol.public_key], 4)], asset_id=tx_1.id, ).sign([carol.private_key]) + assert b.validate_transaction(tx_2) + b.store_bulk_transactions([tx_2]) tx_3 = Transaction.transfer( tx_2.to_inputs()[2:3], @@ -92,20 +103,25 @@ def test_get_spent_issue_1271(b, alice, bob, carol): ([carol.public_key], 3)], asset_id=tx_1.id, ).sign([carol.private_key]) + assert b.validate_transaction(tx_3) + b.store_bulk_transactions([tx_3]) tx_4 = Transaction.transfer( tx_2.to_inputs()[1:2] + tx_3.to_inputs()[0:1], [([bob.public_key], 3)], asset_id=tx_1.id, ).sign([alice.private_key]) + assert b.validate_transaction(tx_4) + b.store_bulk_transactions([tx_4]) tx_5 = Transaction.transfer( tx_2.to_inputs()[0:1], [([alice.public_key], 2)], asset_id=tx_1.id, ).sign([bob.private_key]) - block_5 = b.create_block([tx_1, tx_2, tx_3, tx_4, tx_5]) - b.write_block(block_5) + assert b.validate_transaction(tx_5) + b.store_bulk_transactions([tx_5]) + assert b.get_spent(tx_2.id, 0) == tx_5 assert not b.get_spent(tx_5.id, 0) assert b.get_outputs_filtered(alice.public_key) From abc6db7999d41e68ffa317e8214f94b0d3a85ba7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 18 Jul 2018 15:20:13 +0200 Subject: [PATCH 04/11] Last PR before the release of BigchainDB 2.0 Beta 3 (#2391) * Update CHANGELOG.md for 2.0 Beta 3 release * Update docs re/ installation for 2.0.0b3 * Update version.py and k8s files for 2.0.0b3 --- CHANGELOG.md | 8 ++++++++ bigchaindb/version.py | 4 ++-- docs/server/source/simple-network-setup.md | 5 +++-- k8s/bigchaindb/bigchaindb-ss.yaml | 2 +- k8s/dev-setup/bigchaindb.yaml | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bffdd155..e89ccc85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,14 @@ For reference, the possible headings are: * **Known Issues** * **Notes** +## [2.0 Beta 3] - 2018-07-18 + +Tag name: v2.0.0b3 + +### Fixed + +Fixed a bug in transaction validation. For some more-complex situations, it would say that a valid transaction was invalid. This bug was actually fixed before; it was [issue #1271](https://github.com/bigchaindb/bigchaindb/issues/1271). The unit test for it was turned off while we integrated Tendermint. Then the query implementation code got changed, reintroducing the bug, but the unit test was off so the bug wasn't caught. When we turned the test back on, shortly after releasing Beta 2, it failed, unveiling the bug. [Pull request #2389](https://github.com/bigchaindb/bigchaindb/pull/2389) + ## [2.0 Beta 2] - 2018-07-16 Tag name: v2.0.0b2 diff --git a/bigchaindb/version.py b/bigchaindb/version.py index edc112cd..72f9eecf 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '2.0.0b2' -__short_version__ = '2.0b2' +__version__ = '2.0.0b3' +__short_version__ = '2.0b3' diff --git a/docs/server/source/simple-network-setup.md b/docs/server/source/simple-network-setup.md index 165cf006..752b39ba 100644 --- a/docs/server/source/simple-network-setup.md +++ b/docs/server/source/simple-network-setup.md @@ -52,10 +52,11 @@ BigchainDB Server requires **Python 3.6+**, so make sure your system has it. Ins sudo apt install -y python3-pip libssl-dev ``` -Now install the latest version of BigchainDB. Check the [project page on PyPI][bdb:pypi] for the last version (which was `2.0.0a6` at the time of writing) and install it: +Now install the latest version of BigchainDB. You can find the latest version by going to the [BigchainDB project release history page on PyPI][bdb:pypi]. For example, to install version 2.0.0b3, you would do: ``` -sudo pip3 install bigchaindb==2.0.0a6 +# Change 2.0.0b3 to the latest version as explained above: +sudo pip3 install bigchaindb==2.0.0b3 ``` Check that you installed the correct version of BigchainDB Server using `bigchaindb --version`. diff --git a/k8s/bigchaindb/bigchaindb-ss.yaml b/k8s/bigchaindb/bigchaindb-ss.yaml index 636da377..027c3fed 100644 --- a/k8s/bigchaindb/bigchaindb-ss.yaml +++ b/k8s/bigchaindb/bigchaindb-ss.yaml @@ -154,7 +154,7 @@ spec: timeoutSeconds: 15 # BigchainDB container - name: bigchaindb - image: bigchaindb/bigchaindb:2.0.0-beta2 + image: bigchaindb/bigchaindb:2.0.0-beta3 imagePullPolicy: Always args: - start diff --git a/k8s/dev-setup/bigchaindb.yaml b/k8s/dev-setup/bigchaindb.yaml index e8db3042..d62685fc 100644 --- a/k8s/dev-setup/bigchaindb.yaml +++ b/k8s/dev-setup/bigchaindb.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:2.0.0-beta2 + image: bigchaindb/bigchaindb:2.0.0-beta3 imagePullPolicy: Always args: - start From e681427b757754252fec441e0530940d1f29fb7b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 23 Jul 2018 10:42:36 +0200 Subject: [PATCH 05/11] Problem: The write-a-bep page/process is daunting (#2347) * Problem: The write-a-bep page/process is daunting Solution: Make the BEP writing process easier and more approachable * Problem: BEP should be written uppercase Solution: Change bigchaindb/BEPs label names to use uppercase "BEP" --- .../source/ways-to-contribute/write-a-bep.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/contributing/source/ways-to-contribute/write-a-bep.md b/docs/contributing/source/ways-to-contribute/write-a-bep.md index 182f850c..9bb5e8f4 100644 --- a/docs/contributing/source/ways-to-contribute/write-a-bep.md +++ b/docs/contributing/source/ways-to-contribute/write-a-bep.md @@ -1,7 +1,14 @@ -# Write a BigchaindB Enhancement Proposal (BEP) +# Write a BigchainDB Enhancement Proposal (BEP) -- Review [1/C4](https://github.com/bigchaindb/BEPs/tree/master/1), the process we use to accept any new code or PR of any kind, including one that adds a BEP to `bigchaindb/BEPs`. -- Review [2/COSS](https://github.com/bigchaindb/BEPs/tree/master/2). Maybe print it for reference. It outlines what can go in a BEP. -- Don't spend weeks on your BEP. Version 1 should take up to a few hours to write. You can add to it in the future. The process is iterative. If you need more than a few hours, then consider writing multiple BEPs. -- Do _not_ start writing code before you think about it. You should always write a BEP first. Once you do that, you can start implementing it. To do that, make a pull request and say it implements your BEP. -- Do _not_ write your BEP as an issue (i.e. a GitHub issue). +If you have an idea for a new feature or enhancement, and you want some feedback before you write a full BigchainDB Enhancement Proposal (BEP), then feel free to: + - ask in the [bigchaindb/bigchaindb Gitter chat room](https://gitter.im/bigchaindb/bigchaindb) or + - [open a new issue in the bigchaindb/BEPs repo](https://github.com/bigchaindb/BEPs/issues/new) and give it the label **BEP idea**. + +If you want to discuss an existing BEP, then [open a new issue in the bigchaindb/BEPs repo](https://github.com/bigchaindb/BEPs/issues/new) and give it the label **discuss existing BEP**. + +## Steps to Write a New BEP + +1. Look at the structure of existing BEPs in the [bigchaindb/BEPs repo](https://github.com/bigchaindb/BEPs). Note the section headings. [BEP-2](https://github.com/bigchaindb/BEPs/tree/master/2) (our variant of the consensus-oriented specification system [COSS]) says more about the expected structure and process. +1. Write a first draft of your BEP. It doesn't have to be long or perfect. +1. Push your BEP draft to the [bigchaindb/BEPs repo](https://github.com/bigchaindb/BEPs) and make a pull request. [BEP-1](https://github.com/bigchaindb/BEPs/tree/master/1) (our variant of C4) outlines the process we use to handle all pull requests. In particular, we try to merge all pull requests quickly. +1. Your BEP can be revised by pushing more pull requests. From 42cc0d0e8642d6bafc606dbd50b8648681d0a09c Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 23 Jul 2018 16:54:27 +0200 Subject: [PATCH 06/11] Problem: No troubleshooting help for users of simple-network-setup (#2398) Solution: Add a Troubleshooting section to the simple-network-setup docs --- docs/server/source/simple-network-setup.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/server/source/simple-network-setup.md b/docs/server/source/simple-network-setup.md index 752b39ba..dc13dd43 100644 --- a/docs/server/source/simple-network-setup.md +++ b/docs/server/source/simple-network-setup.md @@ -244,6 +244,19 @@ If you followed the recommended approach and created startup scripts for Bigchai If you followed the above instructions, then your node should be publicly-accessible with BigchainDB Root URL `http://hostname:9984` (where hostname is something like `bdb7.canada.vmsareus.net` or `17.122.200.76`). That is, anyone can interact with your node using the [BigchainDB HTTP API](http-client-server-api.html) exposed at that address. The most common way to do that is to use one of the [BigchainDB Drivers](./drivers-clients/index.html). +## Troubleshooting + +To check which nodes your node is connected to (via Tendermint protocols), do: + +```text +# if you don't jq installed, then install it +sudo apt install jq +# then do +curl -s localhost:26657/net_info | jq ".result.peers[].node_info | {id, listen_addr, moniker}" +``` + +Tendermint has other endpoints besides `/net_info`: see [the Tendermint RPC docs](https://tendermint.github.io/slate/?shell#introduction). + ## Refreshing Your Node If you want to refresh your node back to a fresh empty state, then your best bet is to terminate it and deploy a new virtual machine, but if that's not an option, then you can: From 75c22141ade8cd8caa7b2b1882c793c877a774b4 Mon Sep 17 00:00:00 2001 From: Muawia Khan Date: Mon, 23 Jul 2018 17:38:24 +0200 Subject: [PATCH 07/11] Problem: mongodb repos different for debian and ubuntu (#2385) * Problem: mongodb repos different for debian and ubuntu * Ignore errors for checking bdb process --- pkg/configuration/roles/bigchaindb/tasks/stop.yml | 3 ++- pkg/configuration/roles/mongodb/defaults/main.yml | 3 ++- pkg/configuration/roles/mongodb/tasks/debian.yml | 11 ++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/configuration/roles/bigchaindb/tasks/stop.yml b/pkg/configuration/roles/bigchaindb/tasks/stop.yml index 016a06fe..8c518690 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/stop.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/stop.yml @@ -13,6 +13,7 @@ - name: Get Running BigchainDB Process(es) shell: "ps aux | grep \"[b]igchaindb\" | awk '{print $2}'" register: bdb_ps + ignore_errors: yes when: stack_type|lower == "local" tags: [bigchaindb] @@ -30,4 +31,4 @@ - gunicorn ignore_errors: yes when: stack_type|lower == "local" - tags: [bigchaindb] \ No newline at end of file + tags: [bigchaindb] diff --git a/pkg/configuration/roles/mongodb/defaults/main.yml b/pkg/configuration/roles/mongodb/defaults/main.yml index 612b96b1..e198d632 100644 --- a/pkg/configuration/roles/mongodb/defaults/main.yml +++ b/pkg/configuration/roles/mongodb/defaults/main.yml @@ -8,7 +8,8 @@ distribution_major: "{{ ansible_distribution_major_version }}" server_arch: "amd64,arm64" # MongoDB Repos -mongodb_apt_repo: "deb [arch={{ server_arch }}] http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongo_version }} {{'main' if ansible_distribution == 'debian' else 'multiverse'}}" +mongodb_apt_repo: "deb [arch={{ server_arch }}] http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongo_version }} multiverse" +mongodb_deb_repo: "deb http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongo_version }} main" mongodb_yum_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/$releasever/{{ mongodb_package }}/{{ mongo_version }}/{{ ansible_architecture }}" mongodb_dnf_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/7/{{ mongodb_package }}/{{ mongo_version }}/{{ ansible_architecture }}" diff --git a/pkg/configuration/roles/mongodb/tasks/debian.yml b/pkg/configuration/roles/mongodb/tasks/debian.yml index 86dab821..e2e38855 100644 --- a/pkg/configuration/roles/mongodb/tasks/debian.yml +++ b/pkg/configuration/roles/mongodb/tasks/debian.yml @@ -22,6 +22,15 @@ repo: "{{ mongodb_apt_repo }}" state: present update_cache: no + when: distribution_name == "ubuntu" + tags: [mongodb] + +- name: Add MongoDB repo and update cache | deb + apt_repository: + repo: "{{ mongodb_deb_repo }}" + state: present + update_cache: no + when: distribution_name == "debian" tags: [mongodb] - name: Install MongoDB | apt @@ -31,4 +40,4 @@ update_cache: yes with_items: - "{{ mongodb_package }}" - tags: [mongodb] \ No newline at end of file + tags: [mongodb] From 12740686e94cc39a250c91285fe684ec7e8d51a7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 23 Jul 2018 18:03:21 +0200 Subject: [PATCH 08/11] Problem: Docs say Crypto Conditions are part of ILP (#2399) * Fixed 2 PEP257 compliance errors * Upgrade Tendermint to 0.19.7 except for Docker image, which is still at 0.19.2 * Problem: Docs say Crypto Conditions are part of ILP Solution: Edit the docs so they no longer say that --- docs/root/source/smart-contracts.rst | 2 +- docs/root/source/transaction-concepts.md | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/root/source/smart-contracts.rst b/docs/root/source/smart-contracts.rst index 91b01b3b..b031f2f2 100644 --- a/docs/root/source/smart-contracts.rst +++ b/docs/root/source/smart-contracts.rst @@ -3,7 +3,7 @@ BigchainDB and Smart Contracts One can store the source code of any smart contract (i.e. a computer program) in BigchainDB, but BigchainDB won't run arbitrary smart contracts. -BigchainDB will run the subset of smart contracts expressible using `Crypto-Conditions `_. Crypto-conditions are part of the `Interledger Protocol `_. +BigchainDB will run the subset of smart contracts expressible using `Crypto-Conditions `_. The owners of an asset can impose conditions on it that must be met for the asset to be transferred to new owners. Examples of possible conditions (crypto-conditions) include: diff --git a/docs/root/source/transaction-concepts.md b/docs/root/source/transaction-concepts.md index 18c2d3b7..6a3b183d 100644 --- a/docs/root/source/transaction-concepts.md +++ b/docs/root/source/transaction-concepts.md @@ -27,9 +27,8 @@ and the other output might have 15 oak trees for another set of owners. Each output also has an associated condition: the condition that must be met (by a TRANSFER transaction) to transfer/spend the output. -BigchainDB supports a variety of conditions, -a subset of the [Interledger Protocol (ILP)](https://interledger.org/) -crypto-conditions. For details, see +BigchainDB supports a variety of conditions. +For details, see the section titled **Transaction Components: Conditions** in the relevant [BigchainDB Transactions Spec](https://github.com/bigchaindb/BEPs/tree/master/tx-specs/). From d0a24ef5844b72e93ab93b258de0756f0c9d8504 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 25 Jul 2018 16:16:06 +0200 Subject: [PATCH 10/11] Problem: validation runs redundant queries (#2409) Solution: when a transaction is stored, the system splits it in three components, data, metadata, and the other fields of the transaction. When the system retrieves a transaction, it needs to make three different queries to reconstruct the original transaction. If a transaction does not exist, the system does three queries anyway, even if just one query is enough. This patch fixes this by doing the additional queries if and only if the transaction exists. --- bigchaindb/tendermint/lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index ed7aec15..5e238c94 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -254,10 +254,10 @@ class BigchainDB(object): def get_transaction(self, transaction_id, include_status=False): transaction = backend.query.get_transaction(self.connection, transaction_id) - asset = backend.query.get_asset(self.connection, transaction_id) - metadata = backend.query.get_metadata(self.connection, [transaction_id]) if transaction: + asset = backend.query.get_asset(self.connection, transaction_id) + metadata = backend.query.get_metadata(self.connection, [transaction_id]) if asset: transaction['asset'] = asset From 2386ca9d717accf08b17e92b724d7f87580b1e37 Mon Sep 17 00:00:00 2001 From: Zachary Bowen Date: Wed, 25 Jul 2018 16:59:25 +0200 Subject: [PATCH 11/11] Refactor tendermint directory to project root (#2401) * Problem: core.py contains an unused class, `Bigchain` Solution: Remove core.py. Refactor BigchainDB Class to remove inheritance from Bigchain. * Problem: core.py contains an unused class, `Bigchain` Solution: Remove core.py. Refactor BigchainDB Class to remove inheritance from Bigchain. * Fixed flake8 complaint about too many blank lines * Attempting to fix Sphinx docs. This may result in some redundant commits, as I don't know what I'm doing, and I can't experiment without running the CI... Sorry in advance! * Attempting to fix Sphinx docs. This may result in some redundant commits, as I don't know what I'm doing, and I can't experiment without running the CI... Sorry in advance! * Updating from master changed BigchainDB.process_post_response to a private method, so I had to align with that. * Fixed a couple stale references to bigchaindb.Bigchain in docstrings * Missed a reference to `Bigchain` in a patch call... * Problem: BigchainDB class should be part of project root Solution: Removed the /tendermint directory and moved its contents to project root * Problem: Flake8 complained that imports were not at the top of the file Solution: Had to play around with the order of imports to avoid cyclic dependencies, but its working and style compliant now * Problem: Stale reference to /tendermint directory in the index Solution: Removed the references to /tendermint * Problem: Flake8 complaining of unused import in __init__.py Solution: The import is there so I can import App directly from bigchaindb, rather than from bigchaindb.core (which I think improves code readability. I added a # noqa to silence Flake8. * Problem: Stale references to `bigchaindb.tendermint.BigchainDB` in the rst files cause Sphinx to fail Solution: Updated the autodoc files to use `bigchaindb.BigchainDB` instead * Problem: Stale reference to the `tendermint` directory in an @patch in a disabled test Solution: Updated the @patch for completeness * Problem: BigchainDB class should be part of project root Solution: Removed the /tendermint directory and moved its contents to project root * Problem: Flake8 complained that imports were not at the top of the file Solution: Had to play around with the order of imports to avoid cyclic dependencies, but its working and style compliant now * Problem: Stale reference to /tendermint directory in the index Solution: Removed the references to /tendermint * Problem: Flake8 complaining of unused import in __init__.py Solution: The import is there so I can import App directly from bigchaindb, rather than from bigchaindb.core (which I think improves code readability. I added a # noqa to silence Flake8. * Problem: Stale references to `bigchaindb.tendermint.BigchainDB` in the rst files cause Sphinx to fail Solution: Updated the autodoc files to use `bigchaindb.BigchainDB` instead * Problem: Stale reference to the `tendermint` directory in an @patch in a disabled test Solution: Updated the @patch for completeness --- bigchaindb/README.md | 2 +- bigchaindb/__init__.py | 5 +- bigchaindb/commands/bigchaindb.py | 10 +- bigchaindb/core.py | 178 ++++++++++++++++++ bigchaindb/{tendermint => }/event_stream.py | 2 +- bigchaindb/{tendermint => }/fastquery.py | 0 bigchaindb/{tendermint => }/lib.py | 5 +- bigchaindb/models.py | 4 +- .../{tendermint/commands.py => start.py} | 6 +- bigchaindb/tendermint/__init__.py | 7 - bigchaindb/tendermint/core.py | 178 ------------------ .../utils.py => tendermint_utils.py} | 0 bigchaindb/web/server.py | 2 +- .../generate_http_server_api_documentation.py | 3 +- docs/server/source/appendices/index.rst | 1 - .../appendices/tendermint-integration.rst | 26 --- .../appendices/the-bigchaindb-class.rst | 2 +- tests/backend/localmongodb/test_queries.py | 8 +- tests/commands/test_commands.py | 6 +- tests/conftest.py | 8 +- tests/db/test_bigchain_api.py | 28 +-- tests/tendermint/conftest.py | 2 +- tests/tendermint/test_core.py | 18 +- tests/tendermint/test_event_stream.py | 8 +- tests/tendermint/test_integration.py | 6 +- tests/tendermint/test_lib.py | 6 +- tests/tendermint/test_utils.py | 10 +- tests/test_config_utils.py | 2 +- tests/test_core.py | 6 +- tests/web/conftest.py | 2 +- tests/web/test_block_tendermint.py | 2 +- tests/web/test_outputs.py | 6 +- tests/web/test_transactions.py | 10 +- 33 files changed, 262 insertions(+), 297 deletions(-) rename bigchaindb/{tendermint => }/event_stream.py (97%) rename bigchaindb/{tendermint => }/fastquery.py (100%) rename bigchaindb/{tendermint => }/lib.py (99%) rename bigchaindb/{tendermint/commands.py => start.py} (95%) delete mode 100644 bigchaindb/tendermint/__init__.py delete mode 100644 bigchaindb/tendermint/core.py rename bigchaindb/{tendermint/utils.py => tendermint_utils.py} (100%) delete mode 100644 docs/server/source/appendices/tendermint-integration.rst diff --git a/bigchaindb/README.md b/bigchaindb/README.md index dbb08afb..3ed61495 100644 --- a/bigchaindb/README.md +++ b/bigchaindb/README.md @@ -4,7 +4,7 @@ A high-level description of the files and subdirectories of BigchainDB. ## Files -### [`tendermint/lib.py`](./tendermint/lib.py) +### [`lib.py`](lib.py) The `BigchainDB` class is defined here. Most node-level operations and database interactions are found in this file. This is the place to start if you are interested in implementing a server API, since many of these class methods concern BigchainDB interacting with the outside world. diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 011737fa..05bf85a0 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -2,6 +2,9 @@ import copy import logging from bigchaindb.log import DEFAULT_LOGGING_CONFIG as log_config +from bigchaindb.lib import BigchainDB # noqa +from bigchaindb.version import __version__ # noqa +from bigchaindb.core import App # noqa # from functools import reduce # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 @@ -84,5 +87,3 @@ config = { # the user wants to reconfigure the node. Check ``bigchaindb.config_utils`` # for more info. _config = copy.deepcopy(config) -from bigchaindb.tendermint import BigchainDB # noqa -from bigchaindb.version import __version__ # noqa diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 188cb7b3..379b89a2 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -21,7 +21,7 @@ from bigchaindb.commands import utils from bigchaindb.commands.utils import (configure_bigchaindb, input_on_stderr) from bigchaindb.log import setup_logging -from bigchaindb.tendermint.utils import public_key_from_base64 +from bigchaindb.tendermint_utils import public_key_from_base64 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -97,7 +97,7 @@ def run_configure(args): def run_upsert_validator(args): """Store validators which should be synced with Tendermint""" - b = bigchaindb.tendermint.BigchainDB() + b = bigchaindb.BigchainDB() public_key = public_key_from_base64(args.public_key) validator = {'pub_key': {'type': 'ed25519', 'data': public_key}, @@ -113,7 +113,7 @@ def run_upsert_validator(args): def _run_init(): - bdb = bigchaindb.tendermint.BigchainDB() + bdb = bigchaindb.BigchainDB() schema.init_database(connection=bdb.connection) @@ -170,7 +170,7 @@ def run_start(args): setup_logging() logger.info('BigchainDB Version %s', bigchaindb.__version__) - run_recover(bigchaindb.tendermint.lib.BigchainDB()) + run_recover(bigchaindb.lib.BigchainDB()) try: if not args.skip_initialize_database: @@ -180,7 +180,7 @@ def run_start(args): pass logger.info('Starting BigchainDB main process.') - from bigchaindb.tendermint.commands import start + from bigchaindb.start import start start() diff --git a/bigchaindb/core.py b/bigchaindb/core.py index e69de29b..354a39a0 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -0,0 +1,178 @@ +"""This module contains all the goodness to integrate BigchainDB +with Tendermint.""" +import logging + +from abci.application import BaseApplication +from abci.types_pb2 import ( + ResponseInitChain, + ResponseInfo, + ResponseCheckTx, + ResponseBeginBlock, + ResponseDeliverTx, + ResponseEndBlock, + ResponseCommit, + Validator, + PubKey +) + +from bigchaindb import BigchainDB +from bigchaindb.tendermint_utils import (decode_transaction, + calculate_hash) +from bigchaindb.lib import Block, PreCommitState +from bigchaindb.backend.query import PRE_COMMIT_ID + + +CodeTypeOk = 0 +CodeTypeError = 1 +logger = logging.getLogger(__name__) + + +class App(BaseApplication): + """Bridge between BigchainDB and Tendermint. + + The role of this class is to expose the BigchainDB + transactional logic to the Tendermint Consensus + State Machine.""" + + def __init__(self, bigchaindb=None): + self.bigchaindb = bigchaindb or BigchainDB() + self.block_txn_ids = [] + self.block_txn_hash = '' + self.block_transactions = [] + self.validators = None + self.new_height = None + + def init_chain(self, validators): + """Initialize chain with block of height 0""" + + block = Block(app_hash='', height=0, transactions=[]) + self.bigchaindb.store_block(block._asdict()) + return ResponseInitChain() + + def info(self, request): + """Return height of the latest committed block.""" + r = ResponseInfo() + block = self.bigchaindb.get_latest_block() + if block: + r.last_block_height = block['height'] + r.last_block_app_hash = block['app_hash'].encode('utf-8') + else: + r.last_block_height = 0 + r.last_block_app_hash = b'' + return r + + def check_tx(self, raw_transaction): + """Validate the transaction before entry into + the mempool. + + Args: + raw_tx: a raw string (in bytes) transaction.""" + + logger.benchmark('CHECK_TX_INIT') + logger.debug('check_tx: %s', raw_transaction) + transaction = decode_transaction(raw_transaction) + if self.bigchaindb.is_valid_transaction(transaction): + logger.debug('check_tx: VALID') + logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) + return ResponseCheckTx(code=CodeTypeOk) + else: + logger.debug('check_tx: INVALID') + logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) + return ResponseCheckTx(code=CodeTypeError) + + def begin_block(self, req_begin_block): + """Initialize list of transaction. + Args: + req_begin_block: block object which contains block header + and block hash. + """ + logger.benchmark('BEGIN BLOCK, height:%s, num_txs:%s', + req_begin_block.header.height, + req_begin_block.header.num_txs) + + self.block_txn_ids = [] + self.block_transactions = [] + return ResponseBeginBlock() + + def deliver_tx(self, raw_transaction): + """Validate the transaction before mutating the state. + + Args: + raw_tx: a raw string (in bytes) transaction.""" + logger.debug('deliver_tx: %s', raw_transaction) + transaction = self.bigchaindb.is_valid_transaction( + decode_transaction(raw_transaction), self.block_transactions) + + if not transaction: + logger.debug('deliver_tx: INVALID') + return ResponseDeliverTx(code=CodeTypeError) + else: + logger.debug('storing tx') + self.block_txn_ids.append(transaction.id) + self.block_transactions.append(transaction) + return ResponseDeliverTx(code=CodeTypeOk) + + def end_block(self, request_end_block): + """Calculate block hash using transaction ids and previous block + hash to be stored in the next block. + + Args: + height (int): new height of the chain.""" + + height = request_end_block.height + self.new_height = height + block_txn_hash = calculate_hash(self.block_txn_ids) + block = self.bigchaindb.get_latest_block() + + if self.block_txn_ids: + self.block_txn_hash = calculate_hash([block['app_hash'], block_txn_hash]) + else: + self.block_txn_hash = block['app_hash'] + + validator_updates = self.bigchaindb.get_validator_update() + validator_updates = [encode_validator(v) for v in validator_updates] + + # set sync status to true + self.bigchaindb.delete_validator_update() + + # Store pre-commit state to recover in case there is a crash + # during `commit` + pre_commit_state = PreCommitState(commit_id=PRE_COMMIT_ID, + height=self.new_height, + transactions=self.block_txn_ids) + logger.debug('Updating PreCommitState: %s', self.new_height) + self.bigchaindb.store_pre_commit_state(pre_commit_state._asdict()) + return ResponseEndBlock(validator_updates=validator_updates) + + def commit(self): + """Store the new height and along with block hash.""" + + data = self.block_txn_hash.encode('utf-8') + + # register a new block only when new transactions are received + if self.block_txn_ids: + self.bigchaindb.store_bulk_transactions(self.block_transactions) + block = Block(app_hash=self.block_txn_hash, + height=self.new_height, + transactions=self.block_txn_ids) + # NOTE: storing the block should be the last operation during commit + # this effects crash recovery. Refer BEP#8 for details + self.bigchaindb.store_block(block._asdict()) + + logger.debug('Commit-ing new block with hash: apphash=%s ,' + 'height=%s, txn ids=%s', data, self.new_height, + self.block_txn_ids) + logger.benchmark('COMMIT_BLOCK, height:%s', self.new_height) + return ResponseCommit(data=data) + + +def encode_validator(v): + ed25519_public_key = v['pub_key']['data'] + # NOTE: tendermint expects public to be encoded in go-amino format + + pub_key = PubKey(type='ed25519', + data=bytes.fromhex(ed25519_public_key)) + + return Validator(pub_key=pub_key, + address=b'', + power=v['power']) diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/event_stream.py similarity index 97% rename from bigchaindb/tendermint/event_stream.py rename to bigchaindb/event_stream.py index 9cbd28f2..8a68b7f1 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/event_stream.py @@ -8,7 +8,7 @@ import aiohttp from bigchaindb import config from bigchaindb.common.utils import gen_timestamp from bigchaindb.events import EventTypes, Event -from bigchaindb.tendermint.utils import decode_transaction_base64 +from bigchaindb.tendermint_utils import decode_transaction_base64 HOST = config['tendermint']['host'] diff --git a/bigchaindb/tendermint/fastquery.py b/bigchaindb/fastquery.py similarity index 100% rename from bigchaindb/tendermint/fastquery.py rename to bigchaindb/fastquery.py diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/lib.py similarity index 99% rename from bigchaindb/tendermint/lib.py rename to bigchaindb/lib.py index 5e238c94..80e93bf3 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/lib.py @@ -16,13 +16,12 @@ except ImportError: import requests import bigchaindb -from bigchaindb import backend, config_utils +from bigchaindb import backend, config_utils, fastquery from bigchaindb.models import Transaction from bigchaindb.common.exceptions import (SchemaValidationError, ValidationError, DoubleSpend) -from bigchaindb.tendermint.utils import encode_transaction, merkleroot -from bigchaindb.tendermint import fastquery +from bigchaindb.tendermint_utils import encode_transaction, merkleroot from bigchaindb import exceptions as core_exceptions from bigchaindb.consensus import BaseConsensusRules diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 33689d0c..62c04517 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -14,7 +14,7 @@ class Transaction(Transaction): """Validate transaction spend Args: - bigchain (BigchainDB): an instantiated bigchaindb.tendermint.BigchainDB object. + bigchain (BigchainDB): an instantiated bigchaindb.BigchainDB object. Returns: The transaction (Transaction) if the transaction is valid else it @@ -108,7 +108,7 @@ class Transaction(Transaction): asset from the asset table and reconstructs the transaction. Args: - bigchain (:class:`~bigchaindb.tendermint.BigchainDB`): An instance + bigchain (:class:`~bigchaindb.BigchainDB`): An instance of BigchainDB used to perform database queries. tx_dict_list (:list:`dict` or :obj:`dict`): The transaction dict or list of transaction dict as returned from the database. diff --git a/bigchaindb/tendermint/commands.py b/bigchaindb/start.py similarity index 95% rename from bigchaindb/tendermint/commands.py rename to bigchaindb/start.py index d50f0f31..4bcc8218 100644 --- a/bigchaindb/tendermint/commands.py +++ b/bigchaindb/start.py @@ -3,10 +3,10 @@ import logging import setproctitle import bigchaindb -from bigchaindb.tendermint.lib import BigchainDB -from bigchaindb.tendermint.core import App +from bigchaindb.lib import BigchainDB +from bigchaindb.core import App from bigchaindb.web import server, websocket_server -from bigchaindb.tendermint import event_stream +from bigchaindb import event_stream from bigchaindb.events import Exchange, EventTypes from bigchaindb.utils import Process diff --git a/bigchaindb/tendermint/__init__.py b/bigchaindb/tendermint/__init__.py deleted file mode 100644 index b98dd439..00000000 --- a/bigchaindb/tendermint/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Code necessary for integrating with Tendermint.""" - -# Order is important! -# If we import core first, core will try to load BigchainDB from -# __init__ itself, causing a loop. -from bigchaindb.tendermint.lib import BigchainDB # noqa -from bigchaindb.tendermint.core import App # noqa diff --git a/bigchaindb/tendermint/core.py b/bigchaindb/tendermint/core.py deleted file mode 100644 index e6e43484..00000000 --- a/bigchaindb/tendermint/core.py +++ /dev/null @@ -1,178 +0,0 @@ -"""This module contains all the goodness to integrate BigchainDB -with Tendermint.""" -import logging - -from abci.application import BaseApplication -from abci.types_pb2 import ( - ResponseInitChain, - ResponseInfo, - ResponseCheckTx, - ResponseBeginBlock, - ResponseDeliverTx, - ResponseEndBlock, - ResponseCommit, - Validator, - PubKey -) - -from bigchaindb.tendermint import BigchainDB -from bigchaindb.tendermint.utils import (decode_transaction, - calculate_hash) -from bigchaindb.tendermint.lib import Block, PreCommitState -from bigchaindb.backend.query import PRE_COMMIT_ID - - -CodeTypeOk = 0 -CodeTypeError = 1 -logger = logging.getLogger(__name__) - - -class App(BaseApplication): - """Bridge between BigchainDB and Tendermint. - - The role of this class is to expose the BigchainDB - transactional logic to the Tendermint Consensus - State Machine.""" - - def __init__(self, bigchaindb=None): - self.bigchaindb = bigchaindb or BigchainDB() - self.block_txn_ids = [] - self.block_txn_hash = '' - self.block_transactions = [] - self.validators = None - self.new_height = None - - def init_chain(self, validators): - """Initialize chain with block of height 0""" - - block = Block(app_hash='', height=0, transactions=[]) - self.bigchaindb.store_block(block._asdict()) - return ResponseInitChain() - - def info(self, request): - """Return height of the latest committed block.""" - r = ResponseInfo() - block = self.bigchaindb.get_latest_block() - if block: - r.last_block_height = block['height'] - r.last_block_app_hash = block['app_hash'].encode('utf-8') - else: - r.last_block_height = 0 - r.last_block_app_hash = b'' - return r - - def check_tx(self, raw_transaction): - """Validate the transaction before entry into - the mempool. - - Args: - raw_tx: a raw string (in bytes) transaction.""" - - logger.benchmark('CHECK_TX_INIT') - logger.debug('check_tx: %s', raw_transaction) - transaction = decode_transaction(raw_transaction) - if self.bigchaindb.is_valid_transaction(transaction): - logger.debug('check_tx: VALID') - logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) - return ResponseCheckTx(code=CodeTypeOk) - else: - logger.debug('check_tx: INVALID') - logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) - return ResponseCheckTx(code=CodeTypeError) - - def begin_block(self, req_begin_block): - """Initialize list of transaction. - Args: - req_begin_block: block object which contains block header - and block hash. - """ - logger.benchmark('BEGIN BLOCK, height:%s, num_txs:%s', - req_begin_block.header.height, - req_begin_block.header.num_txs) - - self.block_txn_ids = [] - self.block_transactions = [] - return ResponseBeginBlock() - - def deliver_tx(self, raw_transaction): - """Validate the transaction before mutating the state. - - Args: - raw_tx: a raw string (in bytes) transaction.""" - logger.debug('deliver_tx: %s', raw_transaction) - transaction = self.bigchaindb.is_valid_transaction( - decode_transaction(raw_transaction), self.block_transactions) - - if not transaction: - logger.debug('deliver_tx: INVALID') - return ResponseDeliverTx(code=CodeTypeError) - else: - logger.debug('storing tx') - self.block_txn_ids.append(transaction.id) - self.block_transactions.append(transaction) - return ResponseDeliverTx(code=CodeTypeOk) - - def end_block(self, request_end_block): - """Calculate block hash using transaction ids and previous block - hash to be stored in the next block. - - Args: - height (int): new height of the chain.""" - - height = request_end_block.height - self.new_height = height - block_txn_hash = calculate_hash(self.block_txn_ids) - block = self.bigchaindb.get_latest_block() - - if self.block_txn_ids: - self.block_txn_hash = calculate_hash([block['app_hash'], block_txn_hash]) - else: - self.block_txn_hash = block['app_hash'] - - validator_updates = self.bigchaindb.get_validator_update() - validator_updates = [encode_validator(v) for v in validator_updates] - - # set sync status to true - self.bigchaindb.delete_validator_update() - - # Store pre-commit state to recover in case there is a crash - # during `commit` - pre_commit_state = PreCommitState(commit_id=PRE_COMMIT_ID, - height=self.new_height, - transactions=self.block_txn_ids) - logger.debug('Updating PreCommitState: %s', self.new_height) - self.bigchaindb.store_pre_commit_state(pre_commit_state._asdict()) - return ResponseEndBlock(validator_updates=validator_updates) - - def commit(self): - """Store the new height and along with block hash.""" - - data = self.block_txn_hash.encode('utf-8') - - # register a new block only when new transactions are received - if self.block_txn_ids: - self.bigchaindb.store_bulk_transactions(self.block_transactions) - block = Block(app_hash=self.block_txn_hash, - height=self.new_height, - transactions=self.block_txn_ids) - # NOTE: storing the block should be the last operation during commit - # this effects crash recovery. Refer BEP#8 for details - self.bigchaindb.store_block(block._asdict()) - - logger.debug('Commit-ing new block with hash: apphash=%s ,' - 'height=%s, txn ids=%s', data, self.new_height, - self.block_txn_ids) - logger.benchmark('COMMIT_BLOCK, height:%s', self.new_height) - return ResponseCommit(data=data) - - -def encode_validator(v): - ed25519_public_key = v['pub_key']['data'] - # NOTE: tendermint expects public to be encoded in go-amino format - - pub_key = PubKey(type='ed25519', - data=bytes.fromhex(ed25519_public_key)) - - return Validator(pub_key=pub_key, - address=b'', - power=v['power']) diff --git a/bigchaindb/tendermint/utils.py b/bigchaindb/tendermint_utils.py similarity index 100% rename from bigchaindb/tendermint/utils.py rename to bigchaindb/tendermint_utils.py diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 646f6f17..348d3fec 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -11,7 +11,7 @@ from flask_cors import CORS import gunicorn.app.base from bigchaindb import utils -from bigchaindb.tendermint import BigchainDB +from bigchaindb import BigchainDB from bigchaindb.web.routes import add_routes from bigchaindb.web.strip_content_type_middleware import StripContentTypeMiddleware diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 2256c355..9050d05c 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -5,8 +5,7 @@ import os import os.path from bigchaindb.common.transaction import Transaction, Input, TransactionLink -from bigchaindb.tendermint import BigchainDB -from bigchaindb.tendermint import lib +from bigchaindb import lib from bigchaindb.web import server diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index f96b365a..033a84d2 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -10,7 +10,6 @@ Appendices the-bigchaindb-class backend commands - tendermint-integration aws-setup generate-key-pair-for-ssh firewall-notes diff --git a/docs/server/source/appendices/tendermint-integration.rst b/docs/server/source/appendices/tendermint-integration.rst deleted file mode 100644 index bc800400..00000000 --- a/docs/server/source/appendices/tendermint-integration.rst +++ /dev/null @@ -1,26 +0,0 @@ -###################### -Tendermint Integration -###################### - - -.. automodule:: bigchaindb.tendermint - :special-members: __init__ - -.. automodule:: bigchaindb.tendermint.lib - :special-members: __init__ - :noindex: - -.. automodule:: bigchaindb.tendermint.core - :special-members: __init__ - -.. automodule:: bigchaindb.tendermint.event_stream - :special-members: __init__ - -.. automodule:: bigchaindb.tendermint.fastquery - :special-members: __init__ - -.. automodule:: bigchaindb.tendermint.commands - :special-members: __init__ - -.. automodule:: bigchaindb.tendermint.utils - :special-members: __init__ diff --git a/docs/server/source/appendices/the-bigchaindb-class.rst b/docs/server/source/appendices/the-bigchaindb-class.rst index 0054e3f4..8d604d2c 100644 --- a/docs/server/source/appendices/the-bigchaindb-class.rst +++ b/docs/server/source/appendices/the-bigchaindb-class.rst @@ -2,4 +2,4 @@ The BigchainDB Class #################### -.. autoclass:: bigchaindb.tendermint.BigchainDB +.. autoclass:: bigchaindb.BigchainDB diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index d0cfcbb7..a9e64ac0 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -229,7 +229,7 @@ def test_get_spending_transactions(user_pk, user_sk): def test_store_block(): from bigchaindb.backend import connect, query - from bigchaindb.tendermint.lib import Block + from bigchaindb.lib import Block conn = connect() block = Block(app_hash='random_utxo', @@ -242,7 +242,7 @@ def test_store_block(): def test_get_block(): from bigchaindb.backend import connect, query - from bigchaindb.tendermint.lib import Block + from bigchaindb.lib import Block conn = connect() block = Block(app_hash='random_utxo', @@ -345,7 +345,7 @@ def test_get_unspent_outputs(db_context, utxoset): def test_store_pre_commit_state(db_context): from bigchaindb.backend import query - from bigchaindb.tendermint.lib import PreCommitState + from bigchaindb.lib import PreCommitState state = PreCommitState(commit_id='test', height=3, @@ -359,7 +359,7 @@ def test_store_pre_commit_state(db_context): def test_get_pre_commit_state(db_context): from bigchaindb.backend import query - from bigchaindb.tendermint.lib import PreCommitState + from bigchaindb.lib import PreCommitState state = PreCommitState(commit_id='test2', height=3, diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 17403e8c..578f06b1 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -90,7 +90,7 @@ def test_bigchain_run_init_when_db_exists(mocker, capsys): def test__run_init(mocker): from bigchaindb.commands.bigchaindb import _run_init bigchain_mock = mocker.patch( - 'bigchaindb.commands.bigchaindb.bigchaindb.tendermint.BigchainDB') + 'bigchaindb.commands.bigchaindb.bigchaindb.BigchainDB') init_db_mock = mocker.patch( 'bigchaindb.commands.bigchaindb.schema.init_database', autospec=True, @@ -274,7 +274,7 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, @patch('bigchaindb.config_utils.autoconfigure') @patch('bigchaindb.commands.bigchaindb.run_recover') -@patch('bigchaindb.tendermint.commands.start') +@patch('bigchaindb.start.start') def test_recover_db_on_start(mock_autoconfigure, mock_run_recover, mock_start, @@ -293,7 +293,7 @@ def test_recover_db_on_start(mock_autoconfigure, def test_run_recover(b, alice, bob): from bigchaindb.commands.bigchaindb import run_recover from bigchaindb.models import Transaction - from bigchaindb.tendermint.lib import Block, PreCommitState + from bigchaindb.lib import Block, PreCommitState from bigchaindb.backend.query import PRE_COMMIT_ID from bigchaindb.backend import query diff --git a/tests/conftest.py b/tests/conftest.py index d1ad5368..f4c05197 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ from pymongo import MongoClient from bigchaindb.common import crypto from bigchaindb.log import setup_logging -from bigchaindb.tendermint.lib import Block +from bigchaindb.lib import Block TEST_DB_NAME = 'bigchain_test' @@ -269,13 +269,13 @@ def merlin_pubkey(merlin): @pytest.fixture def b(): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB return BigchainDB() @pytest.fixture def tb(): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB return BigchainDB() @@ -514,7 +514,7 @@ def event_loop(request): @pytest.fixture(scope='session') def abci_server(): from abci import ABCIServer - from bigchaindb.tendermint.core import App + from bigchaindb.core import App from bigchaindb.utils import Process app = ABCIServer(app=App()) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index c285401a..b1ff8594 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -280,7 +280,7 @@ class TestBigchainApi(object): @pytest.mark.usefixtures('inputs') def test_write_transaction(self, b, user_pk, user_sk): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB from bigchaindb.models import Transaction input_tx = b.get_owned_ids(user_pk).pop() @@ -439,7 +439,7 @@ class TestBigchainApi(object): from bigchaindb.common.exceptions import InputDoesNotExist from bigchaindb.common.transaction import Input, TransactionLink from bigchaindb.models import Transaction - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB # Create an input for a non existing transaction input = Input(Ed25519Sha256(public_key=b58decode(user_pk)), @@ -970,8 +970,8 @@ class TestMultipleInputs(object): def test_get_owned_ids_calls_get_outputs_filtered(): - from bigchaindb.tendermint import BigchainDB - with patch('bigchaindb.tendermint.BigchainDB.get_outputs_filtered') as gof: + from bigchaindb import BigchainDB + with patch('bigchaindb.BigchainDB.get_outputs_filtered') as gof: b = BigchainDB() res = b.get_owned_ids('abc') gof.assert_called_once_with('abc', spent=False) @@ -981,13 +981,13 @@ def test_get_owned_ids_calls_get_outputs_filtered(): @pytest.mark.tendermint def test_get_outputs_filtered_only_unspent(): from bigchaindb.common.transaction import TransactionLink - from bigchaindb.tendermint.lib import BigchainDB + from bigchaindb.lib import BigchainDB - go = 'bigchaindb.tendermint.fastquery.FastQuery.get_outputs_by_public_key' + go = 'bigchaindb.fastquery.FastQuery.get_outputs_by_public_key' with patch(go) as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - fs = 'bigchaindb.tendermint.fastquery.FastQuery.filter_spent_outputs' + fs = 'bigchaindb.fastquery.FastQuery.filter_spent_outputs' with patch(fs) as filter_spent: filter_spent.return_value = [TransactionLink('b', 2)] out = BigchainDB().get_outputs_filtered('abc', spent=False) @@ -998,12 +998,12 @@ def test_get_outputs_filtered_only_unspent(): @pytest.mark.tendermint def test_get_outputs_filtered_only_spent(): from bigchaindb.common.transaction import TransactionLink - from bigchaindb.tendermint.lib import BigchainDB - go = 'bigchaindb.tendermint.fastquery.FastQuery.get_outputs_by_public_key' + from bigchaindb.lib import BigchainDB + go = 'bigchaindb.fastquery.FastQuery.get_outputs_by_public_key' with patch(go) as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - fs = 'bigchaindb.tendermint.fastquery.FastQuery.filter_unspent_outputs' + fs = 'bigchaindb.fastquery.FastQuery.filter_unspent_outputs' with patch(fs) as filter_spent: filter_spent.return_value = [TransactionLink('b', 2)] out = BigchainDB().get_outputs_filtered('abc', spent=True) @@ -1012,13 +1012,13 @@ def test_get_outputs_filtered_only_spent(): @pytest.mark.tendermint -@patch('bigchaindb.tendermint.fastquery.FastQuery.filter_unspent_outputs') -@patch('bigchaindb.tendermint.fastquery.FastQuery.filter_spent_outputs') +@patch('bigchaindb.fastquery.FastQuery.filter_unspent_outputs') +@patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') def test_get_outputs_filtered(filter_spent, filter_unspent): from bigchaindb.common.transaction import TransactionLink - from bigchaindb.tendermint.lib import BigchainDB + from bigchaindb.lib import BigchainDB - go = 'bigchaindb.tendermint.fastquery.FastQuery.get_outputs_by_public_key' + go = 'bigchaindb.fastquery.FastQuery.get_outputs_by_public_key' with patch(go) as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] diff --git a/tests/tendermint/conftest.py b/tests/tendermint/conftest.py index aca3c680..2d63d7c4 100644 --- a/tests/tendermint/conftest.py +++ b/tests/tendermint/conftest.py @@ -3,7 +3,7 @@ import pytest @pytest.fixture def b(): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB return BigchainDB() diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py index cb67a065..1a047e7d 100644 --- a/tests/tendermint/test_core.py +++ b/tests/tendermint/test_core.py @@ -6,7 +6,7 @@ from abci.types_pb2 import ( RequestEndBlock ) -from bigchaindb.tendermint.core import CodeTypeOk, CodeTypeError +from bigchaindb.core import CodeTypeOk, CodeTypeError pytestmark = [pytest.mark.tendermint, pytest.mark.bdb] @@ -17,7 +17,7 @@ def encode_tx_to_bytes(transaction): def test_check_tx__signed_create_is_ok(b): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -34,7 +34,7 @@ def test_check_tx__signed_create_is_ok(b): def test_check_tx__unsigned_create_is_error(b): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -51,7 +51,7 @@ def test_check_tx__unsigned_create_is_error(b): @pytest.mark.bdb def test_deliver_tx__valid_create_updates_db(b): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -84,7 +84,7 @@ def test_deliver_tx__valid_create_updates_db(b): def test_deliver_tx__double_spend_fails(b): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -113,7 +113,7 @@ def test_deliver_tx__double_spend_fails(b): def test_deliver_transfer_tx__double_spend_fails(b): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -157,9 +157,9 @@ def test_deliver_transfer_tx__double_spend_fails(b): def test_end_block_return_validator_updates(b): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.backend import query - from bigchaindb.tendermint.core import encode_validator + from bigchaindb.core import encode_validator from bigchaindb.backend.query import VALIDATOR_UPDATE_ID app = App(b) @@ -183,7 +183,7 @@ def test_end_block_return_validator_updates(b): def test_store_pre_commit_state_in_end_block(b, alice): - from bigchaindb.tendermint import App + from bigchaindb import App from bigchaindb.backend import query from bigchaindb.models import Transaction from bigchaindb.backend.query import PRE_COMMIT_ID diff --git a/tests/tendermint/test_event_stream.py b/tests/tendermint/test_event_stream.py index c547a9f2..72a935c6 100644 --- a/tests/tendermint/test_event_stream.py +++ b/tests/tendermint/test_event_stream.py @@ -8,7 +8,7 @@ import pytest @pytest.mark.tendermint def test_process_event_new_block(): - from bigchaindb.tendermint.event_stream import process_event + from bigchaindb.event_stream import process_event event = '{"jsonrpc": "2.0", "id": "test_stream_id#event", "result": {'\ '"query": "tm.event=\'NewBlock\'", "data": { "type": "CF18EA939D3240",'\ @@ -46,7 +46,7 @@ def test_process_event_new_block(): @pytest.mark.tendermint def test_process_event_empty_block(): - from bigchaindb.tendermint.event_stream import process_event + from bigchaindb.event_stream import process_event event = '{"jsonrpc": "2.0", "id": "bigchaindb_stream_1524555674#event",'\ '"result": {"query": "tm.event=\'NewBlock\'", "data": {"type": '\ @@ -67,7 +67,7 @@ def test_process_event_empty_block(): @pytest.mark.tendermint def test_process_unknown_event(): - from bigchaindb.tendermint.event_stream import process_event + from bigchaindb.event_stream import process_event event = '{"jsonrpc": "2.0", "id": "test_stream_id#event",'\ ' "result": { "query": "tm.event=\'UnknownEvent\'" }}' @@ -80,7 +80,7 @@ def test_process_unknown_event(): @pytest.mark.asyncio @pytest.mark.abci async def test_subscribe_events(tendermint_ws_url, b): - from bigchaindb.tendermint.event_stream import subscribe_events + from bigchaindb.event_stream import subscribe_events from bigchaindb.common.crypto import generate_key_pair from bigchaindb.models import Transaction diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py index edb8cfa2..4f6b530b 100644 --- a/tests/tendermint/test_integration.py +++ b/tests/tendermint/test_integration.py @@ -12,8 +12,8 @@ from io import BytesIO @pytest.mark.tendermint @pytest.mark.bdb def test_app(tb): - from bigchaindb.tendermint import App - from bigchaindb.tendermint.utils import calculate_hash + from bigchaindb import App + from bigchaindb.tendermint_utils import calculate_hash from bigchaindb.common.crypto import generate_key_pair from bigchaindb.models import Transaction @@ -103,7 +103,7 @@ def test_upsert_validator(b, alice): from bigchaindb.backend.query import VALIDATOR_UPDATE_ID from bigchaindb.backend import query, connect from bigchaindb.models import Transaction - from bigchaindb.tendermint.utils import public_key_to_base64 + from bigchaindb.tendermint_utils import public_key_to_base64 import time conn = connect() diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 9c296e60..e6aa8f07 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -47,7 +47,7 @@ def test_asset_is_separated_from_transaciton(b): @pytest.mark.bdb def test_get_latest_block(tb): - from bigchaindb.tendermint.lib import Block + from bigchaindb.lib import Block b = tb for i in range(10): @@ -63,7 +63,7 @@ def test_get_latest_block(tb): @pytest.mark.bdb @patch('bigchaindb.backend.query.get_block', return_value=None) -@patch('bigchaindb.tendermint.lib.BigchainDB.get_latest_block', return_value={'height': 10}) +@patch('bigchaindb.BigchainDB.get_latest_block', return_value={'height': 10}) def test_get_empty_block(_0, _1, tb): assert tb.get_block(5) == {'height': 5, 'transactions': []} @@ -86,7 +86,7 @@ def test_validation_error(b): def test_write_and_post_transaction(mock_post, b): from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair - from bigchaindb.tendermint.utils import encode_transaction + from bigchaindb.tendermint_utils import encode_transaction alice = generate_key_pair() tx = Transaction.create([alice.public_key], diff --git a/tests/tendermint/test_utils.py b/tests/tendermint/test_utils.py index 01d91bd6..8e4c7d1f 100644 --- a/tests/tendermint/test_utils.py +++ b/tests/tendermint/test_utils.py @@ -12,7 +12,7 @@ pytestmark = pytest.mark.tendermint def test_encode_decode_transaction(b): - from bigchaindb.tendermint.utils import (encode_transaction, + from bigchaindb.tendermint_utils import (encode_transaction, decode_transaction) asset = { @@ -30,7 +30,7 @@ def test_encode_decode_transaction(b): def test_calculate_hash_no_key(b): - from bigchaindb.tendermint.utils import calculate_hash + from bigchaindb.tendermint_utils import calculate_hash # pass an empty list assert calculate_hash([]) == '' @@ -38,7 +38,7 @@ def test_calculate_hash_no_key(b): # TODO test for the case of an empty list of hashes, and possibly other cases. def test_merkleroot(): - from bigchaindb.tendermint.utils import merkleroot + from bigchaindb.tendermint_utils import merkleroot hashes = [sha3_256(i.encode()).digest() for i in 'abc'] assert merkleroot(hashes) == ( '78c7c394d3158c218916b7ae0ebdea502e0f4e85c08e3b371e3dfd824d389fa3') @@ -54,14 +54,14 @@ SAMPLE_PUBLIC_KEY = { def test_convert_base64_public_key_to_address(): - from bigchaindb.tendermint.utils import public_key64_to_address + from bigchaindb.tendermint_utils import public_key64_to_address address = public_key64_to_address(SAMPLE_PUBLIC_KEY['pub_key']['value']) assert address == SAMPLE_PUBLIC_KEY['address'] def test_public_key_encoding_decoding(): - from bigchaindb.tendermint.utils import (public_key_from_base64, + from bigchaindb.tendermint_utils import (public_key_from_base64, public_key_to_base64) public_key = public_key_from_base64(SAMPLE_PUBLIC_KEY['pub_key']['value']) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 30dcfd60..f5bce81d 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -35,7 +35,7 @@ def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): import bigchaindb from bigchaindb import config_utils from bigchaindb.common import exceptions - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB assert 'CONFIGURED' not in bigchaindb.config # We need to disable ``bigchaindb.config_utils.autoconfigure`` to avoid reading diff --git a/tests/test_core.py b/tests/test_core.py index 5f89baa1..7a648d83 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -31,7 +31,7 @@ def config(request, monkeypatch): @pytest.mark.skipif(reason='will be fixed in another PR') def test_bigchain_class_default_initialization(config): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB from bigchaindb.consensus import BaseConsensusRules from bigchaindb.backend.connection import Connection bigchain = BigchainDB() @@ -44,7 +44,7 @@ def test_bigchain_class_default_initialization(config): @pytest.mark.skipif(reason='will be fixed in another PR') def test_bigchain_class_initialization_with_parameters(config): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB from bigchaindb.backend import connect from bigchaindb.consensus import BaseConsensusRules init_db_kwargs = { @@ -65,7 +65,7 @@ def test_bigchain_class_initialization_with_parameters(config): @pytest.mark.skipif(reason='will be fixed in another PR') def test_get_blocks_status_containing_tx(monkeypatch): from bigchaindb.backend import query as backend_query - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB blocks = [ {'id': 1}, {'id': 2} ] diff --git a/tests/web/conftest.py b/tests/web/conftest.py index efb0f81b..72659f15 100644 --- a/tests/web/conftest.py +++ b/tests/web/conftest.py @@ -4,7 +4,7 @@ import pytest @pytest.fixture def app(request): from bigchaindb.web import server - from bigchaindb.tendermint.lib import BigchainDB + from bigchaindb.lib import BigchainDB if request.config.getoption('--database-backend') == 'localmongodb': app = server.create_app(debug=True, bigchaindb_factory=BigchainDB) diff --git a/tests/web/test_block_tendermint.py b/tests/web/test_block_tendermint.py index 1542ccae..d627c1fd 100644 --- a/tests/web/test_block_tendermint.py +++ b/tests/web/test_block_tendermint.py @@ -1,7 +1,7 @@ import pytest from bigchaindb.models import Transaction -from bigchaindb.tendermint.lib import Block +from bigchaindb.lib import Block BLOCKS_ENDPOINT = '/api/v1/blocks/' diff --git a/tests/web/test_outputs.py b/tests/web/test_outputs.py index d4d9c4b0..39b9d95a 100644 --- a/tests/web/test_outputs.py +++ b/tests/web/test_outputs.py @@ -11,7 +11,7 @@ def test_get_outputs_endpoint(client, user_pk): m = MagicMock() m.txid = 'a' m.output = 0 - with patch('bigchaindb.tendermint.lib.BigchainDB.get_outputs_filtered') as gof: + with patch('bigchaindb.BigchainDB.get_outputs_filtered') as gof: gof.return_value = [m, m] res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk)) assert res.json == [ @@ -27,7 +27,7 @@ def test_get_outputs_endpoint_unspent(client, user_pk): m = MagicMock() m.txid = 'a' m.output = 0 - with patch('bigchaindb.tendermint.lib.BigchainDB.get_outputs_filtered') as gof: + with patch('bigchaindb.BigchainDB.get_outputs_filtered') as gof: gof.return_value = [m] params = '?spent=False&public_key={}'.format(user_pk) res = client.get(OUTPUTS_ENDPOINT + params) @@ -41,7 +41,7 @@ def test_get_outputs_endpoint_spent(client, user_pk): m = MagicMock() m.txid = 'a' m.output = 0 - with patch('bigchaindb.tendermint.lib.BigchainDB.get_outputs_filtered') as gof: + with patch('bigchaindb.BigchainDB.get_outputs_filtered') as gof: gof.return_value = [m] params = '?spent=true&public_key={}'.format(user_pk) res = client.get(OUTPUTS_ENDPOINT + params) diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 7c52ea94..8582ed16 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -372,7 +372,7 @@ def test_transactions_get_list_good(client): asset_id = '1' * 64 - with patch('bigchaindb.tendermint.BigchainDB.get_transactions_filtered', get_txs_patched): + with patch('bigchaindb.BigchainDB.get_transactions_filtered', get_txs_patched): url = TX_ENDPOINT + '?asset_id=' + asset_id assert client.get(url).json == [ ['asset_id', asset_id], @@ -389,7 +389,7 @@ def test_transactions_get_list_good(client): def test_transactions_get_list_bad(client): def should_not_be_called(): assert False - with patch('bigchaindb.tendermint.BigchainDB.get_transactions_filtered', + with patch('bigchaindb.BigchainDB.get_transactions_filtered', lambda *_, **__: should_not_be_called()): # Test asset id validated url = TX_ENDPOINT + '?asset_id=' + '1' * 63 @@ -404,7 +404,7 @@ def test_transactions_get_list_bad(client): @pytest.mark.tendermint def test_return_only_valid_transaction(client): - from bigchaindb.tendermint import BigchainDB + from bigchaindb import BigchainDB def get_transaction_patched(status): def inner(self, tx_id, include_status): @@ -415,12 +415,12 @@ def test_return_only_valid_transaction(client): # UNDECIDED or VALID block, as well as transactions from the backlog. # As the endpoint uses `get_transaction`, we don't have to test # against invalid transactions here. - with patch('bigchaindb.tendermint.BigchainDB.get_transaction', + with patch('bigchaindb.BigchainDB.get_transaction', get_transaction_patched(BigchainDB.TX_UNDECIDED)): url = '{}{}'.format(TX_ENDPOINT, '123') assert client.get(url).status_code == 404 - with patch('bigchaindb.tendermint.BigchainDB.get_transaction', + with patch('bigchaindb.BigchainDB.get_transaction', get_transaction_patched(BigchainDB.TX_IN_BACKLOG)): url = '{}{}'.format(TX_ENDPOINT, '123') assert client.get(url).status_code == 404