From 65b6040e6b6fa9c2e8d61f7f9f47038256ad043b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 8 Aug 2018 16:46:23 +0200 Subject: [PATCH 1/8] Problem: Codecov shield commented-out in README.md (#2421) Solution: Uncomment it, since Codecov is working again ![Codecov](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 77f26884..d9d907a8 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,8 @@ (including pre-release versions) from PyPI, so show the latest GitHub release instead. ---> - [![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases) [![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/BigchainDB/) [![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.org/bigchaindb/bigchaindb) From 4806b815774473b1fc41f10ca607cbb344b2e539 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 9 Aug 2018 11:14:48 +0200 Subject: [PATCH 2/8] Problem: Inconsistent HTTP API docs (#2403) - Make the title of the asset search section consistent with the example (and consistent with the metadata search title and example). - Add a note that one can use `asset?search` or `asset/?search`, and the same for metadata. --- docs/server/source/http-client-server-api.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index b413430b..fd5c121d 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -301,7 +301,7 @@ Assets Currently this endpoint is only supported if using MongoDB. -.. http:get:: /api/v1/assets?search={search} +.. http:get:: /api/v1/assets/?search={search} Return all assets that match a given text search. @@ -310,6 +310,10 @@ Assets The ``id`` of the asset is the same ``id`` of the CREATE transaction that created the asset. + .. note:: + + You can use ``assets/?search`` or ``assets?search``. + If no assets match the text search it returns an empty list. If the text string is empty or the server does not support text search, @@ -425,6 +429,10 @@ Transaction Metadata The ``id`` of the metadata is the same ``id`` of the transaction where it was defined. + .. note:: + + You can use ``metadata/?search`` or ``metadata?search``. + If no metadata objects match the text search it returns an empty list. If the text string is empty or the server does not support text search, From 205e2cf3fd003ab9c54303392383d88a816cdd6f Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 9 Aug 2018 11:16:05 +0200 Subject: [PATCH 3/8] Problem: Immutability page in docs sometimes wrong or controversial (#2435) Solution: Edit that page --- docs/root/source/immutable.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/root/source/immutable.md b/docs/root/source/immutable.md index 8791c04e..c1aeb9f6 100644 --- a/docs/root/source/immutable.md +++ b/docs/root/source/immutable.md @@ -2,13 +2,14 @@ The word _immutable_ means "unchanging over time or unable to be changed." For example, the decimal digits of π are immutable (3.14159…). -The blockchain community often describes blockchains as “immutable.” If we interpret that word literally, it means that blockchain data is unchangeable or permanent, which is absurd. The data _can_ be changed. For example, a plague might drive humanity extinct; the data would then get corrupted over time due to water damage, thermal noise, and the general increase of entropy. In the case of Bitcoin, nothing so drastic is required: a 51% attack will suffice. +The blockchain community often describes blockchains as “immutable.” If we interpret that word literally, it means that blockchain data is unchangeable or permanent, which is absurd. The data _can_ be changed. For example, a plague might drive humanity extinct; the data would then get corrupted over time due to water damage, thermal noise, and the general increase of entropy. It’s true that blockchain data is more difficult to change (or delete) than usual. It's more than just "tamper-resistant" (which implies intent), blockchain data also resists random changes that can happen without any intent, such as data corruption on a hard drive. Therefore, in the context of blockchains, we interpret the word “immutable” to mean *practically* immutable, for all intents and purposes. (Linguists would say that the word “immutable” is a _term of art_ in the blockchain community.) -Blockchain data can achieve immutability in several ways: +Blockchain data can be made immutable in several ways: -1. **Replication.** All data is replicated (copied) to several different places. The replication factor can be set by the consortium. The higher the replication factor, the more difficult it becomes to change or delete all replicas. +1. **No APIs for changing or deleting data.** Blockchain software usually doesn't expose any APIs for changing or deleting the data stored in the blockchain. BigchainDB has no such APIs. This doesn't prevent changes or deletions from happening in _other_ ways; it's just one line of defense. +1. **Replication.** All data is replicated (copied) to several different places. The higher the replication factor, the more difficult it becomes to change or delete all replicas. 1. **Internal watchdogs.** All nodes monitor all changes and if some unallowed change happens, then appropriate action can be taken. 1. **External watchdogs.** A consortium may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For a consortium with publicly-readable data, the public can act as an auditor. 1. **Economic incentives.** Some blockchain systems make it very expensive to change old stored data. Examples include proof-of-work and proof-of-stake systems. BigchainDB doesn't use explicit incentives like those. @@ -17,5 +18,3 @@ Blockchain data can achieve immutability in several ways: 1. **Full or partial backups** may be recorded from time to time, possibly on magnetic tape storage, other blockchains, printouts, etc. 1. **Strong security.** Node owners can adopt and enforce strong security policies. 1. **Node diversity.** Diversity makes it so that no one thing (e.g. natural disaster or operating system bug) can compromise enough of the nodes. See [the section on the kinds of node diversity](diversity.html). - -Some of these things come "for free" as part of the BigchainDB software, and others require some extra effort from the consortium and node owners. From 4636a489186076846be71380a68431114dac8e9f Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 9 Aug 2018 11:16:56 +0200 Subject: [PATCH 4/8] Problem: Glossary no longer referenced by any docs (#2441) Solution: Remove Glossary section from the server docs --- docs/server/source/glossary.rst | 19 ------------------- docs/server/source/index.rst | 1 - 2 files changed, 20 deletions(-) delete mode 100644 docs/server/source/glossary.rst diff --git a/docs/server/source/glossary.rst b/docs/server/source/glossary.rst deleted file mode 100644 index 8fc03ac2..00000000 --- a/docs/server/source/glossary.rst +++ /dev/null @@ -1,19 +0,0 @@ -Glossary -======== - -.. glossary:: - :sorted: - - associative array - A collection of key/value (or name/value) pairs - such that each possible key appears at most once - in the collection. - In JavaScript (and JSON), all objects behave as associative arrays - with string-valued keys. - In Python and .NET, associative arrays are called *dictionaries*. - In Java and Go, they are called *maps*. - In Ruby, they are called *hashes*. - See also: Wikipedia's articles for - `Associative array `_ - and - `Comparison of programming languages (associative array) `_ diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index be9d1a10..f8a4cb96 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -18,5 +18,4 @@ BigchainDB Server Documentation data-models/index k8s-deployment-template/index release-notes - glossary appendices/index From 30115483174b12ba9a9291e97cf6e5600401ce9d Mon Sep 17 00:00:00 2001 From: Muawia Khan Date: Thu, 9 Aug 2018 15:17:46 +0200 Subject: [PATCH 5/8] Problem: remove autodraft from download link (#2454) --- Dockerfile-all-in-one | 2 +- .../run-node-as-processes.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile-all-in-one b/Dockerfile-all-in-one index 6c50d971..0ccde6bb 100644 --- a/Dockerfile-all-in-one +++ b/Dockerfile-all-in-one @@ -21,7 +21,7 @@ RUN apk --update add sudo bash \ RUN apk --update add mongodb monit # Install Tendermint -RUN wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}-autodraft/tendermint_${TM_VERSION}_linux_amd64.zip \ +RUN wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip \ && unzip tendermint_${TM_VERSION}_linux_amd64.zip \ && mv tendermint /usr/local/bin/ \ && rm tendermint_${TM_VERSION}_linux_amd64.zip diff --git a/docs/contributing/source/dev-setup-coding-and-contribution-process/run-node-as-processes.md b/docs/contributing/source/dev-setup-coding-and-contribution-process/run-node-as-processes.md index 4c98578a..d39b2b9f 100644 --- a/docs/contributing/source/dev-setup-coding-and-contribution-process/run-node-as-processes.md +++ b/docs/contributing/source/dev-setup-coding-and-contribution-process/run-node-as-processes.md @@ -23,7 +23,7 @@ Find [the version number of the latest Tendermint release](https://github.com/te ```bash $ sudo apt install -y unzip -$ wget https://github.com/tendermint/tendermint/releases/download/v0.22.8-autodraft/tendermint_0.22.8_linux_amd64.zip +$ wget https://github.com/tendermint/tendermint/releases/download/v0.22.8/tendermint_0.22.8_linux_amd64.zip $ unzip tendermint_0.22.8_linux_amd64.zip $ rm tendermint_0.22.8_linux_amd64.zip $ sudo mv tendermint /usr/local/bin From 37608242611000de055b0602b149519a02d92f99 Mon Sep 17 00:00:00 2001 From: Zachary Bowen Date: Thu, 9 Aug 2018 17:29:21 +0200 Subject: [PATCH 6/8] Create dynamic upsert validator commands (#2446) * Problem: Need a method to initiate a new upsert-validator election Solution: Added subcommand `new` to `upsert_validator` * Problem: Changes to upsert-validator needed to be reflected in the docs Solution: Wrote a section for `upsert-validator new` --- Dockerfile-dev | 1 - bigchaindb/commands/bigchaindb.py | 91 ++++++++++++++----- bigchaindb/lib.py | 1 + bigchaindb/utils.py | 16 +++- .../source/server-reference/bigchaindb-cli.md | 23 +++-- tests/commands/test_commands.py | 74 +++++++++++++-- tests/conftest.py | 27 +++++- 7 files changed, 187 insertions(+), 46 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 05e4bf98..4425e0d2 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -34,4 +34,3 @@ COPY . /usr/src/app/ WORKDIR /usr/src/app RUN pip install --no-cache-dir --process-dependency-links -e .[dev] RUN bigchaindb -y configure - diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 379b89a2..63394df1 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -9,19 +9,19 @@ import copy import json import sys +from bigchaindb.utils import load_node_key from bigchaindb.common.exceptions import (DatabaseAlreadyExists, DatabaseDoesNotExist, - MultipleValidatorOperationError) + OperationError) import bigchaindb -from bigchaindb import backend +from bigchaindb import backend, ValidatorElection, BigchainDB from bigchaindb.backend import schema from bigchaindb.backend import query -from bigchaindb.backend.query import VALIDATOR_UPDATE_ID, PRE_COMMIT_ID +from bigchaindb.backend.query import PRE_COMMIT_ID from bigchaindb.commands import utils from bigchaindb.commands.utils import (configure_bigchaindb, input_on_stderr) from bigchaindb.log import setup_logging -from bigchaindb.tendermint_utils import public_key_from_base64 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -95,21 +95,48 @@ def run_configure(args): @configure_bigchaindb def run_upsert_validator(args): - """Store validators which should be synced with Tendermint""" + """Initiate and manage elections to change the validator set""" - b = bigchaindb.BigchainDB() - public_key = public_key_from_base64(args.public_key) - validator = {'pub_key': {'type': 'ed25519', - 'data': public_key}, - 'power': args.power} - validator_update = {'validator': validator, - 'update_id': VALIDATOR_UPDATE_ID} - try: - query.store_validator_update(b.connection, validator_update) - except MultipleValidatorOperationError: - logger.error('A validator update is pending to be applied. ' - 'Please re-try after the current update has ' - 'been processed.') + b = BigchainDB() + + # Call the function specified by args.action, as defined above + globals()[f'run_upsert_validator_{args.action}'](args, b) + + +def run_upsert_validator_new(args, bigchain): + """Initiates an election to add/update/remove a validator to an existing BigchainDB network + + :param args: dict + args = { + 'public_key': the public key of the proposed peer, (str) + 'power': the proposed validator power for the new peer, (str) + 'node_id': the node_id of the new peer (str) + 'sk': the path to the private key of the node calling the election (str) + } + :param bigchain: an instance of BigchainDB + :return: election_id (tx_id) + :raises: OperationError if the write transaction fails for any reason + """ + + new_validator = { + 'public_key': args.public_key, + 'power': args.power, + 'node_id': args.node_id + } + + key = load_node_key(args.sk) + + voters = ValidatorElection.recipients(bigchain) + + election = ValidatorElection.generate([key.public_key], + voters, + new_validator, None).sign([key.private_key]) + election.validate(bigchain) + resp = bigchain.write_transaction(election, 'broadcast_tx_commit') + if resp == (202, ''): + return election.id + else: + raise OperationError('Failed to commit election') def _run_init(): @@ -208,16 +235,30 @@ def create_parser(): help='The backend to use. It can only be ' '"localmongodb", currently.') + # parser for managing validator elections validator_parser = subparsers.add_parser('upsert-validator', - help='Add/update/delete a validator') + help='Add/update/delete a validator.') - validator_parser.add_argument('public_key', - help='Public key of the validator.') + validator_subparser = validator_parser.add_subparsers(title='Action', + dest='action') - validator_parser.add_argument('power', - type=int, - help='Voting power of the validator. ' - 'Setting it to 0 will delete the validator.') + new_election_parser = validator_subparser.add_parser('new', + help='Calls a new election.') + + new_election_parser.add_argument('public_key', + help='Public key of the validator to be added/updated/removed.') + + new_election_parser.add_argument('power', + type=int, + help='The proposed power for the validator. ' + 'Setting to 0 will remove the validator.') + + new_election_parser.add_argument('node_id', + help='The node_id of the validator.') + + new_election_parser.add_argument('--private-key', + dest='sk', + help='Path to the private key of the election initiator.') # parsers for showing/exporting config values subparsers.add_parser('show-config', diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py index 12ab9dc8..a0b7e727 100644 --- a/bigchaindb/lib.py +++ b/bigchaindb/lib.py @@ -440,6 +440,7 @@ class BigchainDB(object): validators = result['validators'] for v in validators: v.pop('address') + v['voting_power'] = int(v['voting_power']) return validators diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 8c4180d0..ba781214 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -2,9 +2,13 @@ import contextlib import threading import queue import multiprocessing as mp +import json import setproctitle +from bigchaindb.tendermint_utils import key_from_base64 +from bigchaindb.common.crypto import key_pair_from_ed25519_key + class ProcessGroup(object): @@ -31,7 +35,8 @@ class ProcessGroup(object): class Process(mp.Process): """Wrapper around multiprocessing.Process that uses setproctitle to set the name of the process when running - the target task.""" + the target task. + """ def run(self): setproctitle.setproctitle(self.name) @@ -167,3 +172,12 @@ class Lazy: self.stack = [] return last + + +# Load Tendermint's public and private key from the file path +def load_node_key(path): + with open(path) as json_data: + priv_validator = json.load(json_data) + priv_key = priv_validator['priv_key']['value'] + hex_private_key = key_from_base64(priv_key) + return key_pair_from_ed25519_key(hex_private_key) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 16dbee68..439e7160 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -80,22 +80,27 @@ configuration file as documented under **This is an experimental feature. Users are advised not to use it in production.** -Add, update, or remove a validator from the validators set of the local node. The command implements [3/UPSERT-VALIDATORS](https://github.com/bigchaindb/BEPs/tree/master/3), check it out if you need more details on how this is orchestrated. +Manage elections to add, update, or remove a validator from the validators set of the local node. The upsert-validator subcommands implement [BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21). Check it out if you need more details on how this is orchestrated. -Below is the command line syntax, +Election management is broken into several subcommands. Below is the command line syntax for each, + +#### upsert-validator new + +Calls a new election, proposing a change to the validator set. + +Below is the command line syntax and the return value, ```bash -$ bigchaindb upsert-validator PUBLIC_KEY_OF_VALIDATOR POWER +$ bigchaindb upsert-validator new E_PUBKEY E_POWER E_NODE_ID --private-key PATH_TO_YOUR_PRIVATE_KEY + ``` +Here, `E_PUBKEY`, `E_POWER`, and `E_NODE_ID` are the public key, proposed power, and node id of the validator being voted on. `--private-key` should be the path to wherever the private key for your validator node is stored, (*not* the private key itself.). For example, to add a new validator, provide the public key and node id for some node not already in the validator set, along with whatever voting power you'd like them to have. To remove an existing validator, provide their public key and node id, and set `E_POWER` to `0`. + Example usage, ```bash -$ bigchaindb upsert-validator B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014 10 +$ bigchaindb upsert-validator new B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014 1 12345 --private-key /home/user/.tendermint/config/priv_validator.json ``` -If the command is returns without any error then a request to update the validator set has been successfully submitted. So, even if the command has been successfully executed it doesn't imply that the validator set has been updated. In order to check whether the change has been applied, the node operator can execute `curl http://node_ip:9984/api/v1/validators` which will list the current validators set. Refer to the [validators](/http-client-server-api.html#validators) section of the HTTP API docs for more detail. - -Note: -- When `POWER`is set to `0` then the validator will be removed from the validator set. -- Upsert requests are handled once per block i.e. the validators set is updated once a new block is committed. So, the node operator is not allowed to submit a new upsert request until the current request has been processed. Furthermore, if Tendermint is started with `--consensus.create_empty_blocks=false`, and there are no new incoming transactions then the validators set update is delayed until any new transactions are received and a new block can be committed. +If the command succeeds, it will create an election and return an `election_id`. Elections consist of one vote token per voting power, issued to the members of the validator set. Validators can cast their votes to approve the change to the validator set by spending their vote tokens. The status of the election can be monitored by providing the `election_id` to the `show` subcommand. diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a7c6418b..10e41205 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -18,7 +18,8 @@ def test_make_sure_we_dont_remove_any_command(): assert parser.parse_args(['init']).command assert parser.parse_args(['drop']).command assert parser.parse_args(['start']).command - assert parser.parse_args(['upsert-validator', 'TEMP_PUB_KEYPAIR', '10']).command + assert parser.parse_args(['upsert-validator', 'new', 'TEMP_PUB_KEYPAIR', '10', 'TEMP_NODE_ID', + '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command @pytest.mark.tendermint @@ -341,15 +342,70 @@ class MockResponse(): return {'result': {'latest_block_height': self.height}} +# @pytest.mark.execute +# @patch('bigchaindb.lib.BigchainDB.get_validators') +# @pytest.mark.abci @pytest.mark.skip -@patch('bigchaindb.config_utils.autoconfigure') -@patch('bigchaindb.backend.query.store_validator_update') +def test_upsert_validator_new_with_tendermint(b, priv_validator_path, user_sk, monkeypatch): + """WIP: Will be fixed and activated in the next PR + """ + from bigchaindb.commands.bigchaindb import run_upsert_validator_new + import time + + time.sleep(3) + + def mock_get(): + return [ + {'pub_key': {'value': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=', + 'type': 'tendermint/PubKeyEd25519'}, + 'voting_power': 10} + ] + + # b.get_validators = mock_get + # mock_get_validators = mock_get + # monkeypatch.setattr('requests.get', mock_get) + + proposer_key = b.get_validators()[0]['pub_key']['value'] + + args = Namespace(action='new', + public_key=proposer_key, + power=1, + node_id='12345', + sk=priv_validator_path, + config={}) + resp = run_upsert_validator_new(args, b) + time.sleep(3) + + assert b.get_transaction(resp) + + @pytest.mark.tendermint -def test_upsert_validator(mock_autoconfigure, mock_store_validator_update): - from bigchaindb.commands.bigchaindb import run_upsert_validator +@pytest.mark.bdb +def test_upsert_validator_new_without_tendermint(b, priv_validator_path, user_sk, monkeypatch): + from bigchaindb.commands.bigchaindb import run_upsert_validator_new - args = Namespace(public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', - power='10', config={}) - run_upsert_validator(args) + def mock_get(): + return [ + {'pub_key': {'value': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=', + 'type': 'tendermint/PubKeyEd25519'}, + 'voting_power': 10} + ] - assert mock_store_validator_update.called + def mock_write(tx, mode): + b.store_transaction(tx) + return (202, '') + + b.get_validators = mock_get + b.write_transaction = mock_write + + monkeypatch.setattr('requests.get', mock_get) + + args = Namespace(action='new', + public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', + power=1, + node_id='12345', + sk=priv_validator_path, + config={}) + resp = run_upsert_validator_new(args, b) + + assert b.get_transaction(resp) diff --git a/tests/conftest.py b/tests/conftest.py index 66d85d3a..7cb33b14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,11 @@ Tasks: 1. setup test database before starting the tests 2. delete test database after running the tests """ - +import json import os import copy import random +import tempfile from collections import namedtuple from logging import getLogger from logging.config import dictConfig @@ -668,3 +669,27 @@ def node_keys(): '83VINXdj2ynOHuhvSZz5tGuOE5oYzIi0mEximkX1KYMlt/Csu8JUjA4+by2Pz3fqSLshhuYYeM+IpvqcBl6BEA==', 'PecJ58SaNRsWJZodDmqjpCWqG6btdwXFHLyE40RYlYM=': 'uz8bYgoL4rHErWT1gjjrnA+W7bgD/uDQWSRKDmC8otc95wnnxJo1GxYlmh0OaqOkJaobpu13BcUcvITjRFiVgw=='} + + +@pytest.fixture(scope='session') +def priv_validator_path(node_keys): + (public_key, private_key) = list(node_keys.items())[0] + priv_validator = { + 'address': '84F787D95E196DC5DE5F972666CFECCA36801426', + 'pub_key': { + 'type': 'AC26791624DE60', + 'value': public_key + }, + 'last_height': 0, + 'last_round': 0, + 'last_step': 0, + 'priv_key': { + 'type': '954568A3288910', + 'value': private_key + } + } + fd, path = tempfile.mkstemp() + socket = os.fdopen(fd, 'w') + json.dump(priv_validator, socket) + socket.close() + return path From bbf5310ac85056277e2e4c1cdc1842370674cf45 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 10 Aug 2018 11:39:44 +0200 Subject: [PATCH 7/8] Problem: community drivers not working w/ latest BDB (#2443) Solution: In the docs page listing the Community-Driven Libraries and Tools, make it clear that they might not work with the latest version of BigchainDB. Make it a warning rather than a note. --- docs/server/source/drivers-clients/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 3cf4d0fa..79954971 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -11,10 +11,11 @@ Libraries and Tools Maintained by the BigchainDB Team Community-Driven Libraries and Tools ------------------------------------ -.. note:: +.. warning:: Some of these projects are a work in progress, but may still be useful. + Others might not work with the latest version of BigchainDB. * `Haskell transaction builder `_ * `Go driver `_ From 3092beb9956644bc8e2c972e3072c7eb83c103ef Mon Sep 17 00:00:00 2001 From: Zachary Bowen Date: Mon, 13 Aug 2018 09:54:45 +0200 Subject: [PATCH 8/8] Reenable test bigchain api - part 1 (#2415) * Problem: core.py contains an unused class, `Bigchain` Solution: Remove core.py. Refactor BigchainDB Class to remove inheritance from Bigchain. * Fixed flake8 complaint about too many blank lines * Attempting to fix Sphinx docs. This may result in some redundant commits, as I don't know what I'm doing, and I can't experiment without running the CI... Sorry in advance! * Updating from master changed BigchainDB.process_post_response to a private method, so I had to align with that. * Fixed a couple stale references to bigchaindb.Bigchain in docstrings * Missed a reference to `Bigchain` in a patch call... * Problem: Tests in class `TestBigChainAPI` from `tests/db/test_bigchain_api.py` were disabled Solution: Fixed the tests that still made sense, and deleted the rest. * Problem: Flake8 style complaints Solution: Made things prettier * Problem: Debugging comments were slightly NSFW Solution: Deleted the debugging comments * Problem: Stale use of `pytest.mark.genesis` decorator on these tests Solution: Removed the decorator * Problem: CriticalDoubleInclusion exception was no longer used anywhere in the code Solution: Deleted it * Problem: Unused exception `GenesisBlockAlreadyExistsError` Solution: Deleted it * Problem: I removed a test that covered writing to the DB and checking that everything is stored faithfully Solution: Replaced it and got it working again --- bigchaindb/common/exceptions.py | 4 - bigchaindb/exceptions.py | 4 - tests/conftest.py | 8 +- tests/db/test_bigchain_api.py | 385 ++++---------------------------- 4 files changed, 53 insertions(+), 348 deletions(-) diff --git a/bigchaindb/common/exceptions.py b/bigchaindb/common/exceptions.py index 0c145d13..357e998d 100644 --- a/bigchaindb/common/exceptions.py +++ b/bigchaindb/common/exceptions.py @@ -96,10 +96,6 @@ class ThresholdTooDeep(ValidationError): """Raised if threshold condition is too deep""" -class GenesisBlockAlreadyExistsError(ValidationError): - """Raised when trying to create the already existing genesis block""" - - class MultipleValidatorOperationError(ValidationError): """Raised when a validator update pending but new request is submited""" diff --git a/bigchaindb/exceptions.py b/bigchaindb/exceptions.py index 336ce231..bb5d7c01 100644 --- a/bigchaindb/exceptions.py +++ b/bigchaindb/exceptions.py @@ -4,7 +4,3 @@ class BigchainDBError(Exception): class CriticalDoubleSpend(BigchainDBError): """Data integrity error that requires attention""" - - -class CriticalDoubleInclusion(BigchainDBError): - """Data integrity error that requires attention""" diff --git a/tests/conftest.py b/tests/conftest.py index 7cb33b14..e08a3886 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -330,17 +330,19 @@ def _get_height(b): def inputs(user_pk, b, alice): from bigchaindb.models import Transaction # create blocks with transactions for `USER` to spend - for block in range(4): + for height in range(1, 4): transactions = [ Transaction.create( [alice_pubkey(alice)], [([user_pk], 1)], metadata={'msg': random.random()}, - ).sign([alice_privkey(alice)]).to_dict() + ).sign([alice_privkey(alice)]) for _ in range(10) ] - block = Block(app_hash='', height=_get_height(b), transactions=transactions) + tx_ids = [tx.id for tx in transactions] + block = Block(app_hash='hash'+str(height), height=height, transactions=tx_ids) b.store_block(block._asdict()) + b.store_bulk_transactions(transactions) @pytest.fixture diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 2a8ace6a..88d7d445 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -7,196 +7,74 @@ from base58 import b58decode pytestmark = pytest.mark.bdb -@pytest.mark.skipif(reason='Some tests throw a ResourceWarning that might result in some weird ' - 'exceptions while running the tests. The problem seems to *not* ' - 'interfere with the correctness of the tests. ') -def test_remove_unclosed_sockets(): - pass - - class TestBigchainApi(object): - @pytest.mark.genesis - def test_get_last_voted_block_cyclic_blockchain(self, b, monkeypatch, alice): - from bigchaindb.common.crypto import PrivateKey - from bigchaindb.common.exceptions import CyclicBlockchainError - from bigchaindb.common.utils import serialize - from bigchaindb.models import Transaction - - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) - tx = tx.sign([alice.private_key]) - monkeypatch.setattr('time.time', lambda: 1) - block1 = b.create_block([tx]) - b.write_block(block1) - - # Manipulate vote to create a cyclic Blockchain - vote = b.vote(block1.id, b.get_last_voted_block().id, True) - vote['vote']['previous_block'] = block1.id - vote_data = serialize(vote['vote']) - vote['signature'] = PrivateKey(alice.private_key).sign(vote_data.encode()) - b.write_vote(vote) - - with pytest.raises(CyclicBlockchainError): - b.get_last_voted_block() - - @pytest.mark.genesis - def test_try_voting_while_constructing_cyclic_blockchain(self, b, - monkeypatch, alice): - from bigchaindb.common.exceptions import CyclicBlockchainError - from bigchaindb.models import Transaction - - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) - tx = tx.sign([alice.private_key]) - block1 = b.create_block([tx]) - - # We can simply submit twice the same block id and check if `Bigchain` - # throws - with pytest.raises(CyclicBlockchainError): - b.vote(block1.id, block1.id, True) - - @pytest.mark.genesis - def test_has_previous_vote_when_already_voted(self, b, monkeypatch, alice): + @pytest.mark.tendermint + def test_get_spent_with_double_inclusion_detected(self, b, alice): from bigchaindb.models import Transaction + from bigchaindb.backend.exceptions import OperationError tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) tx = tx.sign([alice.private_key]) - monkeypatch.setattr('time.time', lambda: 1) - block = b.create_block([tx]) - b.write_block(block) + b.store_bulk_transactions([tx]) - assert b.has_previous_vote(block.id) is False - - vote = b.vote(block.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - assert b.has_previous_vote(block.id) is True - - @pytest.mark.genesis - def test_get_spent_with_double_inclusion_detected(self, b, monkeypatch, alice): - from bigchaindb.exceptions import CriticalDoubleInclusion - from bigchaindb.models import Transaction - - tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) - tx = tx.sign([alice.private_key]) - - monkeypatch.setattr('time.time', lambda: 1000000000) - block1 = b.create_block([tx]) - b.write_block(block1) - - monkeypatch.setattr('time.time', lambda: 1000000020) transfer_tx = Transaction.transfer(tx.to_inputs(), [([alice.public_key], 1)], asset_id=tx.id) transfer_tx = transfer_tx.sign([alice.private_key]) - block2 = b.create_block([transfer_tx]) - b.write_block(block2) + b.store_bulk_transactions([transfer_tx]) - monkeypatch.setattr('time.time', lambda: 1000000030) transfer_tx2 = Transaction.transfer(tx.to_inputs(), [([alice.public_key], 1)], asset_id=tx.id) transfer_tx2 = transfer_tx2.sign([alice.private_key]) - block3 = b.create_block([transfer_tx2]) - b.write_block(block3) + with pytest.raises(OperationError): + b.store_bulk_transactions([transfer_tx2]) - # Vote both block2 and block3 valid - vote = b.vote(block2.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - vote = b.vote(block3.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - with pytest.raises(CriticalDoubleInclusion): - b.get_spent(tx.id, 0) - - @pytest.mark.genesis - def test_get_spent_with_double_spend_detected(self, b, monkeypatch, alice): - from bigchaindb.exceptions import CriticalDoubleSpend + @pytest.mark.tendermint + def test_get_spent_with_double_spend_detected(self, b, alice): from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import DoubleSpend + from bigchaindb.exceptions import CriticalDoubleSpend tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) tx = tx.sign([alice.private_key]) - monkeypatch.setattr('time.time', lambda: 1000000000) - block1 = b.create_block([tx]) - b.write_block(block1) + b.store_bulk_transactions([tx]) - monkeypatch.setattr('time.time', lambda: 1000000020) transfer_tx = Transaction.transfer(tx.to_inputs(), [([alice.public_key], 1)], asset_id=tx.id) transfer_tx = transfer_tx.sign([alice.private_key]) - block2 = b.create_block([transfer_tx]) - b.write_block(block2) - - monkeypatch.setattr('time.time', lambda: 1000000030) transfer_tx2 = Transaction.transfer(tx.to_inputs(), [([alice.public_key], 2)], asset_id=tx.id) transfer_tx2 = transfer_tx2.sign([alice.private_key]) - block3 = b.create_block([transfer_tx2]) - b.write_block(block3) - # Vote both block2 and block3 valid - vote = b.vote(block2.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - vote = b.vote(block3.id, b.get_last_voted_block().id, True) - b.write_vote(vote) + with pytest.raises(DoubleSpend): + b.validate_transaction(transfer_tx2, [transfer_tx]) + + b.store_bulk_transactions([transfer_tx]) + + with pytest.raises(DoubleSpend): + b.validate_transaction(transfer_tx2) + + b.store_bulk_transactions([transfer_tx2]) with pytest.raises(CriticalDoubleSpend): b.get_spent(tx.id, 0) - @pytest.mark.genesis - def test_get_block_status_for_tx_with_double_inclusion(self, b, monkeypatch, alice): - from bigchaindb.exceptions import CriticalDoubleInclusion + @pytest.mark.tendermint + def test_get_block_status_for_tx_with_double_inclusion(self, b, alice): from bigchaindb.models import Transaction + from bigchaindb.backend.exceptions import OperationError tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) tx = tx.sign([alice.private_key]) - monkeypatch.setattr('time.time', lambda: 1000000000) - block1 = b.create_block([tx]) - b.write_block(block1) + b.store_bulk_transactions([tx]) - monkeypatch.setattr('time.time', lambda: 1000000020) - block2 = b.create_block([tx]) - b.write_block(block2) + with pytest.raises(OperationError): + b.store_bulk_transactions([tx]) - # Vote both blocks valid (creating a double spend) - vote = b.vote(block1.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - vote = b.vote(block2.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - with pytest.raises(CriticalDoubleInclusion): - b.get_blocks_status_containing_tx(tx.id) - - @pytest.mark.genesis - def test_get_transaction_in_invalid_and_valid_block(self, monkeypatch, b, alice): - from bigchaindb.models import Transaction - - monkeypatch.setattr('time.time', lambda: 1000000000) - tx1 = Transaction.create([alice.public_key], [([alice.public_key], 1)], - metadata={'msg': 1}) - tx1 = tx1.sign([alice.private_key]) - block1 = b.create_block([tx1]) - b.write_block(block1) - - monkeypatch.setattr('time.time', lambda: 1000000020) - tx2 = Transaction.create([alice.public_key], [([alice.public_key], 1)], - metadata={'msg': 2}) - tx2 = tx2.sign([alice.private_key]) - block2 = b.create_block([tx2]) - b.write_block(block2) - - # vote the first block invalid - vote = b.vote(block1.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - - # vote the second block valid - vote = b.vote(block2.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - assert b.get_transaction(tx1.id) is None - assert b.get_transaction(tx2.id) == tx2 - - @pytest.mark.genesis + @pytest.mark.tendermint def test_text_search(self, b, alice): from bigchaindb.models import Transaction from bigchaindb.backend.exceptions import OperationError @@ -215,13 +93,8 @@ class TestBigchainApi(object): tx3 = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset=asset3).sign([alice.private_key]) - # create the block - block = b.create_block([tx1, tx2, tx3]) - b.write_block(block) - - # vote valid - vote = b.vote(block.id, b.get_last_voted_block().id, True) - b.write_vote(vote) + # write the transactions to the DB + b.store_bulk_transactions([tx1, tx2, tx3]) # get the assets through text search # this query only works with MongoDB @@ -232,182 +105,13 @@ class TestBigchainApi(object): else: assert len(assets) == 3 - @pytest.mark.genesis - def test_text_search_returns_valid_only(self, monkeypatch, b, alice): - from bigchaindb.models import Transaction - from bigchaindb.backend.exceptions import OperationError - from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection - - asset_valid = {'msg': 'Hello BigchainDB!'} - asset_invalid = {'msg': 'Goodbye BigchainDB!'} - - monkeypatch.setattr('time.time', lambda: 1000000000) - tx1 = Transaction.create([alice.public_key], [([alice.public_key], 1)], - asset=asset_valid) - tx1 = tx1.sign([alice.private_key]) - block1 = b.create_block([tx1]) - b.write_block(block1) - - monkeypatch.setattr('time.time', lambda: 1000000020) - tx2 = Transaction.create([alice.public_key], [([alice.public_key], 1)], - asset=asset_invalid) - tx2 = tx2.sign([alice.private_key]) - block2 = b.create_block([tx2]) - b.write_block(block2) - - # vote the first block valid - vote = b.vote(block1.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - # vote the second block invalid - vote = b.vote(block2.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - - # get assets with text search - try: - assets = list(b.text_search('bigchaindb')) - except OperationError: - assert not isinstance(b.connection, LocalMongoDBConnection) - return - - # should only return one asset - assert len(assets) == 1 - # should return the asset created by tx1 - assert assets[0] == { - 'data': {'msg': 'Hello BigchainDB!'}, - 'id': tx1.id - } - - @pytest.mark.usefixtures('inputs') - def test_write_transaction(self, b, user_pk, user_sk): - from bigchaindb.models import Transaction - - input_tx = b.get_owned_ids(user_pk).pop() - input_tx = b.get_transaction(input_tx.txid) - inputs = input_tx.to_inputs() - tx = Transaction.transfer(inputs, [([user_pk], 1)], - asset_id=input_tx.id) - tx = tx.sign([user_sk]) - b.write_transaction(tx) - - tx_from_db, status = b.get_transaction(tx.id, include_status=True) - - assert tx_from_db.to_dict() == tx.to_dict() - - @pytest.mark.usefixtures('inputs') - def test_read_transaction(self, b, user_pk, user_sk): - from bigchaindb.models import Transaction - - input_tx = b.get_owned_ids(user_pk).pop() - input_tx = b.get_transaction(input_tx.txid) - inputs = input_tx.to_inputs() - tx = Transaction.transfer(inputs, [([user_pk], 1)], - asset_id=input_tx.id) - tx = tx.sign([user_sk]) - b.write_transaction(tx) - - # create block and write it to the bighcain before retrieving the transaction - block = b.create_block([tx]) - b.write_block(block) - - response, status = b.get_transaction(tx.id, include_status=True) - # add validity information, which will be returned - assert tx.to_dict() == response.to_dict() - - @pytest.mark.usefixtures('inputs') - def test_read_transaction_invalid_block(self, b, user_pk, user_sk): - from bigchaindb.models import Transaction - - input_tx = b.get_owned_ids(user_pk).pop() - input_tx = b.get_transaction(input_tx.txid) - inputs = input_tx.to_inputs() - tx = Transaction.transfer(inputs, [([user_pk], 1)], - asset_id=input_tx.id) - tx = tx.sign([user_sk]) - # There's no need to b.write_transaction(tx) to the backlog - - # create block - block = b.create_block([tx]) - b.write_block(block) - - # vote the block invalid - vote = b.vote(block.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - response = b.get_transaction(tx.id) - - # should be None, because invalid blocks are ignored - # and a copy of the tx is not in the backlog - assert response is None - - @pytest.mark.usefixtures('inputs') - def test_genesis_block(self, b): - from bigchaindb.backend import query - block = query.get_genesis_block(b.connection) - - assert len(block['block']['transactions']) == 1 - assert block['block']['transactions'][0]['operation'] == 'GENESIS' - assert block['block']['transactions'][0]['inputs'][0]['fulfills'] is None - - @pytest.mark.genesis - def test_create_genesis_block_fails_if_table_not_empty(self, b): - from bigchaindb.common.exceptions import GenesisBlockAlreadyExistsError - - with pytest.raises(GenesisBlockAlreadyExistsError): - b.create_genesis_block() - - @pytest.mark.skipif(reason='This test may not make sense after changing the chainification mode') - def test_get_last_block(self, b): - from bigchaindb.backend import query - # get the number of blocks - num_blocks = query.count_blocks(b.connection) - - # get the last block - last_block = b.get_last_block() - - assert last_block['block']['block_number'] == num_blocks - 1 - - @pytest.mark.skipif(reason='This test may not make sense after changing the chainification mode') - def test_get_last_block_id(self, b): - last_block = b.get_last_block() - last_block_id = b.get_last_block_id() - - assert last_block_id == last_block['id'] - - @pytest.mark.skipif(reason='This test may not make sense after changing the chainification mode') - def test_get_previous_block(self, b): - last_block = b.get_last_block() - new_block = b.create_block([]) - b.write_block(new_block) - - prev_block = b.get_previous_block(new_block) - - assert prev_block == last_block - - @pytest.mark.skipif(reason='This test may not make sense after changing the chainification mode') - def test_get_previous_block_id(self, b): - last_block = b.get_last_block() - new_block = b.create_block([]) - b.write_block(new_block) - - prev_block_id = b.get_previous_block_id(new_block) - - assert prev_block_id == last_block['id'] - - def test_create_empty_block(self, b): - from bigchaindb.common.exceptions import OperationError - - with pytest.raises(OperationError) as excinfo: - b.create_block([]) - - assert excinfo.value.args[0] == 'Empty block creation is not allowed' - @pytest.mark.usefixtures('inputs') + @pytest.mark.tendermint def test_non_create_input_not_found(self, b, user_pk): from cryptoconditions import Ed25519Sha256 from bigchaindb.common.exceptions import InputDoesNotExist from bigchaindb.common.transaction import Input, TransactionLink from bigchaindb.models import Transaction - from bigchaindb import BigchainDB # Create an input for a non existing transaction input = Input(Ed25519Sha256(public_key=b58decode(user_pk)), @@ -415,21 +119,28 @@ class TestBigchainApi(object): TransactionLink('somethingsomething', 0)) tx = Transaction.transfer([input], [([user_pk], 1)], asset_id='mock_asset_link') - with pytest.raises(InputDoesNotExist): - tx.validate(BigchainDB()) + tx.validate(b) - def test_count_backlog(self, b, user_pk, alice): - from bigchaindb.backend import query + @pytest.mark.tendermint + def test_write_transaction(self, b, user_sk, user_pk, alice, create_tx): from bigchaindb.models import Transaction - for i in range(4): - tx = Transaction.create([alice.public_key], [([user_pk], 1)], - metadata={'msg': i}) \ - .sign([alice.private_key]) - b.write_transaction(tx) + asset1 = {'msg': 'BigchainDB 1'} - assert query.count_backlog(b.connection) == 4 + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset1).sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + tx_from_db = b.get_transaction(tx.id) + + before = tx.to_dict() + after = tx_from_db.to_dict() + + assert before['asset']['data'] == after['asset']['data'] + before.pop('asset', None) + after.pop('asset', None) + assert before == after class TestTransactionValidation(object):