From e6893632dc0a74c8bb92f10ba410ffb16fd194ca Mon Sep 17 00:00:00 2001 From: Zachary Bowen Date: Mon, 17 Sep 2018 13:59:57 +0200 Subject: [PATCH 01/13] Create migration election class (#2535). * Problem: We need a way to synchronize a halt to block production to allow for upgrades across breaking changes. * Solution: Created `MigrationElection`. * Problem: Need documentation for `migration` elections. * Solution: Updated the docs. * Problem: `MigrationElection` needs 'new' CLI method. * Solution: Updated the definition of `election` to include the new `migration` type. * Problem: The way `end_block` checks for concluded elections assumes there is only one type of election (so we can't conclude an `upsert-validator` and a `chain-migration` at the same height). * Solution: Re-engineered the code in `Elections` to conclude multiple elections in the same block. If more than one election change the validator set, only one of them is applied. * Problem: Tendermint change to store validator changes at height h+2 will break `Election.get_status`. * Solution: Reworked `get_validator_change` to look at only the latest block height or less. --- bigchaindb/__init__.py | 2 + bigchaindb/commands/bigchaindb.py | 60 ++++--- bigchaindb/commands/election_types.py | 9 + bigchaindb/common/schema/__init__.py | 3 + ...saction_chain_migration_election_v2.0.yaml | 44 +++++ .../common/schema/transaction_v2.0.yaml | 1 + bigchaindb/core.py | 15 +- bigchaindb/elections/election.py | 57 ++++--- .../migrations/chain_migration_election.py | 15 ++ .../upsert_validator/validator_election.py | 14 +- .../source/server-reference/bigchaindb-cli.md | 31 +++- tests/commands/test_commands.py | 40 ++++- tests/conftest.py | 140 ++++++++++++++- tests/elections/__init__.py | 0 tests/elections/test_election.py | 64 +++++++ tests/migrations/test_migration_election.py | 9 + tests/upsert_validator/conftest.py | 72 +------- .../test_upsert_validator_vote.py | 161 ++++++++---------- .../test_validator_election.py | 18 +- tests/utils.py | 19 +++ 20 files changed, 551 insertions(+), 223 deletions(-) create mode 100644 bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml create mode 100644 bigchaindb/migrations/chain_migration_election.py create mode 100644 tests/elections/__init__.py create mode 100644 tests/elections/test_election.py create mode 100644 tests/migrations/test_migration_election.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index e1762f5d..223d7a0e 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -7,6 +7,7 @@ import logging from bigchaindb.log import DEFAULT_LOGGING_CONFIG as log_config from bigchaindb.lib import BigchainDB # noqa +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection from bigchaindb.version import __version__ # noqa from bigchaindb.core import App # noqa @@ -99,4 +100,5 @@ from bigchaindb.elections.vote import Vote # noqa Transaction.register_type(Transaction.CREATE, models.Transaction) Transaction.register_type(Transaction.TRANSFER, models.Transaction) Transaction.register_type(ValidatorElection.OPERATION, ValidatorElection) +Transaction.register_type(ChainMigrationElection.OPERATION, ChainMigrationElection) Transaction.register_type(Vote.OPERATION, Vote) diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 96d555bb..f9588dcd 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -13,6 +13,7 @@ import copy import json import sys +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection from bigchaindb.utils import load_node_key from bigchaindb.common.exceptions import (DatabaseDoesNotExist, ValidationError) @@ -115,6 +116,31 @@ def run_election_new(args, bigchain): globals()[f'run_election_new_{args.election_type}'](args, bigchain) +def create_new_election(sk, bigchain, election_class, data): + + try: + key = load_node_key(sk) + voters = election_class.recipients(bigchain) + election = election_class.generate([key.public_key], + voters, + data, None).sign([key.private_key]) + election.validate(bigchain) + except ValidationError as e: + logger.error(e) + return False + except FileNotFoundError as fd_404: + logger.error(fd_404) + return False + + resp = bigchain.write_transaction(election, 'broadcast_tx_commit') + if resp == (202, ''): + logger.info('[SUCCESS] Submitted proposal with id: {}'.format(election.id)) + return election.id + else: + logger.error('Failed to commit election proposal') + return False + + def run_election_new_upsert_validator(args, bigchain): """Initiates an election to add/update/remove a validator to an existing BigchainDB network @@ -136,27 +162,21 @@ def run_election_new_upsert_validator(args, bigchain): 'node_id': args.node_id } - try: - key = load_node_key(args.sk) - voters = ValidatorElection.recipients(bigchain) - election = ValidatorElection.generate([key.public_key], - voters, - new_validator, None).sign([key.private_key]) - election.validate(bigchain) - except ValidationError as e: - logger.error(e) - return False - except FileNotFoundError as fd_404: - logger.error(fd_404) - return False + return create_new_election(args.sk, bigchain, ValidatorElection, new_validator) - resp = bigchain.write_transaction(election, 'broadcast_tx_commit') - if resp == (202, ''): - logger.info('[SUCCESS] Submitted proposal with id: {}'.format(election.id)) - return election.id - else: - logger.error('Failed to commit election proposal') - return False + +def run_election_new_chain_migration(args, bigchain): + """Initiates an election to halt block production + + :param args: dict + args = { + 'sk': the path to the private key of the node calling the election (str) + } + :param bigchain: an instance of BigchainDB + :return: election_id or `False` in case of failure + """ + + return create_new_election(args.sk, bigchain, ChainMigrationElection, {}) def run_election_approve(args, bigchain): diff --git a/bigchaindb/commands/election_types.py b/bigchaindb/commands/election_types.py index b57522f6..4745b85d 100644 --- a/bigchaindb/commands/election_types.py +++ b/bigchaindb/commands/election_types.py @@ -16,5 +16,14 @@ elections = { 'help': 'Path to the private key of the election initiator.' } } + }, + 'chain-migration': { + 'help': 'Call for a halt to block production to allow for a version change across breaking changes.', + 'args': { + '--private-key': { + 'dest': 'sk', + 'help': 'Path to the private key of the election initiator.' + } + } } } diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 25943675..6280b7b9 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -37,6 +37,9 @@ _, TX_SCHEMA_TRANSFER = _load_schema('transaction_transfer_' + _, TX_SCHEMA_VALIDATOR_ELECTION = _load_schema('transaction_validator_election_' + TX_SCHEMA_VERSION) +_, TX_SCHEMA_CHAIN_MIGRATION_ELECTION = _load_schema('transaction_chain_migration_election_' + + TX_SCHEMA_VERSION) + _, TX_SCHEMA_VOTE = _load_schema('transaction_vote_' + TX_SCHEMA_VERSION) diff --git a/bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml b/bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml new file mode 100644 index 00000000..40fa1a7b --- /dev/null +++ b/bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml @@ -0,0 +1,44 @@ +# Copyright BigchainDB GmbH and BigchainDB contributors +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Chain Migration Election Schema - Propose a halt in block production to allow for a version change +required: +- operation +- asset +- outputs +properties: + operation: + type: string + value: "CHAIN_MIGRATION_ELECTION" + asset: + additionalProperties: false + properties: + data: + additionalProperties: false + properties: + seed: + type: string + required: + - data + outputs: + type: array + items: + "$ref": "#/definitions/output" +definitions: + output: + type: object + properties: + condition: + type: object + required: + - uri + properties: + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" diff --git a/bigchaindb/common/schema/transaction_v2.0.yaml b/bigchaindb/common/schema/transaction_v2.0.yaml index 562c0d86..6c056f7f 100644 --- a/bigchaindb/common/schema/transaction_v2.0.yaml +++ b/bigchaindb/common/schema/transaction_v2.0.yaml @@ -63,6 +63,7 @@ definitions: - CREATE - TRANSFER - VALIDATOR_ELECTION + - CHAIN_MIGRATION_ELECTION - VOTE asset: type: object diff --git a/bigchaindb/core.py b/bigchaindb/core.py index f936bd91..8be3cdaf 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -20,13 +20,13 @@ from abci.types_pb2 import ( ) from bigchaindb import BigchainDB +from bigchaindb.elections.election import Election from bigchaindb.version import __tm_supported_versions__ from bigchaindb.utils import tendermint_version_is_compatible from bigchaindb.tendermint_utils import (decode_transaction, calculate_hash) from bigchaindb.lib import Block, PreCommitState from bigchaindb.backend.query import PRE_COMMIT_ID -from bigchaindb.upsert_validator import ValidatorElection import bigchaindb.upsert_validator.validator_utils as vutils from bigchaindb.events import EventTypes, Event @@ -219,15 +219,12 @@ class App(BaseApplication): else: self.block_txn_hash = block['app_hash'] - # Check if the current block concluded any validator elections and - # update the locally tracked validator set - validator_update = ValidatorElection.approved_update(self.bigchaindb, - self.new_height, - self.block_transactions) - update = [validator_update] if validator_update else [] + # Process all concluded elections in the current block and get any update to the validator set + update = Election.approved_elections(self.bigchaindb, + self.new_height, + self.block_transactions) - # Store pre-commit state to recover in case there is a crash - # during `commit` + # Store pre-commit state to recover in case there is a crash during `commit` pre_commit_state = PreCommitState(commit_id=PRE_COMMIT_ID, height=self.new_height, transactions=self.block_txn_ids) diff --git a/bigchaindb/elections/election.py b/bigchaindb/elections/election.py index b78e1456..ca0a4be3 100644 --- a/bigchaindb/elections/election.py +++ b/bigchaindb/elections/election.py @@ -1,6 +1,7 @@ # Copyright BigchainDB GmbH and BigchainDB contributors # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) # Code is Apache-2.0 and docs are CC-BY-4.0 +from collections import defaultdict import base58 from uuid import uuid4 @@ -33,17 +34,18 @@ class Election(Transaction): INCONCLUSIVE = 'inconclusive' # Vote ratio to approve an election ELECTION_THRESHOLD = 2 / 3 + CHANGES_VALIDATOR_SET = True @classmethod - def get_validator_change(cls, bigchain, height=None): - """Return the latest change to the validator set + def get_validator_change(cls, bigchain): + """Return the validator set from the most recent approved block :return: { 'height': , - 'validators': , - 'election_id': + 'validators': } """ + height = bigchain.get_latest_block()['height'] return bigchain.get_validator_change(height) @classmethod @@ -242,24 +244,41 @@ class Election(Transaction): return response @classmethod - def approved_update(cls, bigchain, new_height, txns): - votes = {} - for txn in txns: - if not isinstance(txn, Vote): + def approved_elections(cls, bigchain, new_height, txns): + elections = defaultdict(list) + for tx in txns: + if not isinstance(tx, Vote): + continue + election_id = tx.asset['id'] + elections[election_id].append(tx) + + validator_set_updated = False + validator_set_change = [] + for election_id, votes in elections.items(): + election = Election.has_concluded(bigchain, election_id, votes, new_height) + + if not election: continue - election_id = txn.asset['id'] - election_votes = votes.get(election_id, []) - election_votes.append(txn) - votes[election_id] = election_votes + if election.makes_validator_set_change(): + if validator_set_updated: + continue + validator_set_change.append(election.get_validator_set_change(bigchain, new_height)) + validator_set_updated = True - election = cls.has_concluded(bigchain, election_id, election_votes, new_height) - # Once an election concludes any other conclusion for the same - # or any other election is invalidated - if election: - cls.store_election_results(bigchain, election, new_height) - return cls.on_approval(bigchain, election, new_height) - return None + election.on_approval(bigchain, election, new_height) + + return validator_set_change + + def makes_validator_set_change(self): + return self.CHANGES_VALIDATOR_SET + + def get_validator_set_change(self, bigchain, new_height): + if self.makes_validator_set_change(): + return self.change_validator_set(bigchain, new_height) + + def change_validator_set(self, bigchain, new_height): + raise NotImplementedError @classmethod def on_approval(cls, bigchain, election, new_height): diff --git a/bigchaindb/migrations/chain_migration_election.py b/bigchaindb/migrations/chain_migration_election.py new file mode 100644 index 00000000..e1e783d0 --- /dev/null +++ b/bigchaindb/migrations/chain_migration_election.py @@ -0,0 +1,15 @@ +from bigchaindb.common.schema import TX_SCHEMA_CHAIN_MIGRATION_ELECTION +from bigchaindb.elections.election import Election + + +class ChainMigrationElection(Election): + + OPERATION = 'CHAIN_MIGRATION_ELECTION' + CREATE = OPERATION + ALLOWED_OPERATIONS = (OPERATION,) + TX_SCHEMA_CUSTOM = TX_SCHEMA_CHAIN_MIGRATION_ELECTION + CHANGES_VALIDATOR_SET = False + + @classmethod + def on_approval(cls, bigchain, election, new_height): + bigchain.migrate_abci_chain() diff --git a/bigchaindb/upsert_validator/validator_election.py b/bigchaindb/upsert_validator/validator_election.py index 3daf22eb..856b31a1 100644 --- a/bigchaindb/upsert_validator/validator_election.py +++ b/bigchaindb/upsert_validator/validator_election.py @@ -4,7 +4,7 @@ from bigchaindb.common.exceptions import InvalidPowerChange from bigchaindb.elections.election import Election -from bigchaindb.common.schema import (TX_SCHEMA_VALIDATOR_ELECTION) +from bigchaindb.common.schema import TX_SCHEMA_VALIDATOR_ELECTION from .validator_utils import (new_validator_set, encode_validator, validate_asset_public_key) @@ -36,14 +36,18 @@ class ValidatorElection(Election): super(ValidatorElection, cls).validate_schema(tx) validate_asset_public_key(tx['asset']['data']['public_key']) - @classmethod - def on_approval(cls, bigchain, election, new_height): + def change_validator_set(self, bigchain, new_height): # The new validator set comes into effect from height = new_height+1 - validator_updates = [election.asset['data']] + # (upcoming changes to Tendermint will change this to height = new_height+2) + validator_updates = [self.asset['data']] curr_validator_set = bigchain.get_validators(new_height) updated_validator_set = new_validator_set(curr_validator_set, validator_updates) updated_validator_set = [v for v in updated_validator_set if v['voting_power'] > 0] bigchain.store_validator_set(new_height+1, updated_validator_set) - return encode_validator(election.asset['data']) + return encode_validator(self.asset['data']) + + @classmethod + def on_approval(cls, bigchain, election, new_height): + pass diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 44fa2687..82870445 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -91,6 +91,10 @@ Election management is broken into several subcommands. Below is the command lin Create a new election which proposes a change to your BigChainDB network. +If the command succeeds, it will create an election and return an `election_id`. + +**NOTE**: The election proposal consists of vote tokens allocated to each current validator as per their voting power. Validators then cast their votes to approve the election by spending their vote tokens, (see the documentation on `election approve`). + There are multiple types of election, which each take different parameters. Below is a short description of each type of election, as well as their command line syntax and the return value. ###### election new upsert-validator @@ -106,7 +110,9 @@ $ bigchaindb election new upsert-validator E_PUBKEY E_POWER E_NODE_ID --private- - `E_PUBKEY`: Public key of the node to be added/updated/removed. - `E_POWER`: The new power for the `E_PUBKEY`. NOTE, if power is set to `0` then `E_PUBKEY` will be removed from the validator set when the election concludes. - `E_NODE_ID`: Node id of `E_PUBKEY`. The node operator of `E_PUBKEY` can generate the node id via `tendermint show_node_id`. -- `--private-key`: The path to Tendermint's private key which can be generally found at `/home/user/.tendermint/config/priv_validator.json`. For example, to add a new validator, provide the public key and node id for some node not already in the validator set, along with whatever voting power you'd like them to have. To remove an existing validator, provide their public key and node id, and set `E_POWER` to `0`. Please note that the private key provided here is of the node which is generating this election i.e. +- `--private-key`: The path to Tendermint's private key which can be generally found at `/home/user/.tendermint/config/priv_validator.json`. + +For example, to add a new validator, provide the public key and node id for some node not already in the validator set, along with whatever voting power you'd like them to have. To remove an existing validator, provide their public key and node id, and set `E_POWER` to `0`. Please note that the private key provided here is of the node which is generating this election i.e. NOTE: A change to the validator set can only be proposed by one of the exisitng validators. @@ -118,10 +124,29 @@ $ bigchaindb election new upsert-validator HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52 [SUCCESS] Submitted proposal with id: 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa ``` -If the command succeeds, it will create an election and return an `election_id`. A successful execution of the above command **doesn't** imply that the validator set will be immediately updated but rather it means the proposal has been succcessfully accepted by the network. Once the `election_id` has been generated the node operator should share this `election_id` with other validators in the network and urge them to approve the proposal. Note that the node operator should themsleves also approve the proposal. +A successful execution of the above command **doesn't** imply that the validator set will be immediately updated but rather it means the proposal has been succcessfully accepted by the network. Once the `election_id` has been generated the node operator should share this `election_id` with other validators in the network and urge them to approve the proposal. Note that the node operator should themsleves also approve the proposal. -**NOTE**: The election proposal consists of vote tokens allocated to each current validator as per their voting power. Validators then cast their votes to approve the change to the validator set by spending their vote tokens. +###### election new migration + +Create an election to halt block production, to allow for a version change across breaking changes. + + +```bash +$ bigchaindb election new migration --private-key PATH_TO_YOUR_PRIVATE_KEY +[SUCCESS] Submitted proposal with id: +``` + +- `--private-key`: The path to Tendermint's private key which can be generally found at `/home/user/.tendermint/config/priv_validator.json`. + +Example usage, + +```bash +$ bigchaindb election new migration --private-key /home/user/.tendermint/config/priv_validator.json +[SUCCESS] Submitted proposal with id: 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa +``` + +**NOTE** `migration` elections will halt block production at whichever blockheight they are approved. Once the election is concluded, the validators will need to restart their systems with a new `chain_id` to resume normal operations. #### election approve diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 38e251d8..bbeea227 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -26,6 +26,8 @@ def test_make_sure_we_dont_remove_any_command(): assert parser.parse_args(['start']).command assert parser.parse_args(['election', 'new', 'upsert-validator', 'TEMP_PUB_KEYPAIR', '10', 'TEMP_NODE_ID', '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command + assert parser.parse_args(['election', 'new', 'chain-migration', + '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command assert parser.parse_args(['election', 'approve', 'ELECTION_ID', '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command assert parser.parse_args(['election', 'show', 'ELECTION_ID']).command @@ -341,6 +343,42 @@ def test_election_new_upsert_validator_without_tendermint(caplog, b, priv_valida assert b.get_transaction(election_id) +@pytest.mark.abci +def test_election_new_chain_migration_with_tendermint(b, priv_validator_path, user_sk, validators): + from bigchaindb.commands.bigchaindb import run_election_new_chain_migration + + new_args = Namespace(action='new', + election_type='migration', + sk=priv_validator_path, + config={}) + + election_id = run_election_new_chain_migration(new_args, b) + + assert b.get_transaction(election_id) + + +@pytest.mark.bdb +def test_election_new_chain_migration_without_tendermint(caplog, b, priv_validator_path, user_sk): + from bigchaindb.commands.bigchaindb import run_election_new_chain_migration + + def mock_write(tx, mode): + b.store_bulk_transactions([tx]) + return (202, '') + + b.get_validators = mock_get_validators + b.write_transaction = mock_write + + args = Namespace(action='new', + election_type='migration', + sk=priv_validator_path, + config={}) + + with caplog.at_level(logging.INFO): + election_id = run_election_new_chain_migration(args, b) + assert caplog.records[0].msg == '[SUCCESS] Submitted proposal with id: ' + election_id + assert b.get_transaction(election_id) + + @pytest.mark.bdb def test_election_new_upsert_validator_invalid_election(caplog, b, priv_validator_path, user_sk): from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator @@ -415,7 +453,7 @@ def test_election_approve_without_tendermint(caplog, b, priv_validator_path, new b, election_id = call_election(b, new_validator, node_key) - # call run_upsert_validator_approve with args that point to the election + # call run_election_approve with args that point to the election args = Namespace(action='approve', election_id=election_id, sk=priv_validator_path, diff --git a/tests/conftest.py b/tests/conftest.py index 15cfcf01..4c271aad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,15 +20,17 @@ from logging.config import dictConfig import pytest from pymongo import MongoClient +from bigchaindb import ValidatorElection from bigchaindb.common import crypto from bigchaindb.log import setup_logging +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection from bigchaindb.tendermint_utils import key_from_base64 -from bigchaindb.backend import schema +from bigchaindb.backend import schema, query from bigchaindb.common.crypto import (key_pair_from_ed25519_key, public_key_from_ed25519_key) from bigchaindb.common.exceptions import DatabaseDoesNotExist from bigchaindb.lib import Block - +from tests.utils import gen_vote TEST_DB_NAME = 'bigchain_test' @@ -242,6 +244,26 @@ def b(): return BigchainDB() +@pytest.fixture +def b_mock(b, network_validators): + b.get_validators = mock_get_validators(network_validators) + + return b + + +def mock_get_validators(network_validators): + def validator_set(height): + validators = [] + for public_key, power in network_validators.items(): + validators.append({ + 'public_key': {'type': 'ed25519-base64', 'value': public_key}, + 'voting_power': power + }) + return validators + + return validator_set + + @pytest.fixture def create_tx(alice, user_pk): from bigchaindb.models import Transaction @@ -674,3 +696,117 @@ def new_validator(): 'type': 'ed25519-base16'}, 'power': power, 'node_id': node_id} + + +@pytest.fixture +def valid_upsert_validator_election(b_mock, node_key, new_validator): + voters = ValidatorElection.recipients(b_mock) + return ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +@pytest.fixture +def valid_upsert_validator_election_2(b_mock, node_key, new_validator): + voters = ValidatorElection.recipients(b_mock) + return ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +@pytest.fixture +def valid_chain_migration_election(b_mock, node_key): + voters = ChainMigrationElection.recipients(b_mock) + return ChainMigrationElection.generate([node_key.public_key], + voters, + {}, None).sign([node_key.private_key]) + + +@pytest.fixture +def valid_chain_migration_election_2(b_mock, node_key): + voters = ChainMigrationElection.recipients(b_mock) + return ChainMigrationElection.generate([node_key.public_key], + voters, + {}, None).sign([node_key.private_key]) + + +@pytest.fixture +def ongoing_validator_election(b, valid_upsert_validator_election, ed25519_node_keys): + validators = b.get_validators(height=1) + genesis_validators = {'validators': validators, + 'height': 0, + 'election_id': None} + query.store_validator_set(b.connection, genesis_validators) + + b.store_bulk_transactions([valid_upsert_validator_election]) + block_1 = Block(app_hash='hash_1', height=1, transactions=[valid_upsert_validator_election.id]) + b.store_block(block_1._asdict()) + return valid_upsert_validator_election + + +@pytest.fixture +def ongoing_validator_election_2(b, valid_upsert_validator_election_2, ed25519_node_keys): + validators = b.get_validators(height=1) + genesis_validators = {'validators': validators, + 'height': 0, + 'election_id': None} + query.store_validator_set(b.connection, genesis_validators) + + b.store_bulk_transactions([valid_upsert_validator_election_2]) + block_1 = Block(app_hash='hash_2', height=1, transactions=[valid_upsert_validator_election_2.id]) + b.store_block(block_1._asdict()) + return valid_upsert_validator_election_2 + + +@pytest.fixture +def ongoing_chain_migration_election(b, valid_chain_migration_election, ed25519_node_keys): + + b.store_bulk_transactions([valid_chain_migration_election]) + block_1 = Block(app_hash='hash_1', height=1, transactions=[valid_chain_migration_election.id]) + b.store_block(block_1._asdict()) + return valid_chain_migration_election + + +@pytest.fixture +def ongoing_chain_migration_election_2(b, valid_chain_migration_election_2, ed25519_node_keys): + + b.store_bulk_transactions([valid_chain_migration_election_2]) + block_1 = Block(app_hash='hash_2', height=1, transactions=[valid_chain_migration_election_2.id]) + b.store_block(block_1._asdict()) + return valid_chain_migration_election_2 + + +@pytest.fixture +def validator_election_votes(b_mock, ongoing_validator_election, ed25519_node_keys): + voters = ValidatorElection.recipients(b_mock) + votes = generate_votes(ongoing_validator_election, voters, ed25519_node_keys) + return votes + + +@pytest.fixture +def validator_election_votes_2(b_mock, ongoing_validator_election_2, ed25519_node_keys): + voters = ValidatorElection.recipients(b_mock) + votes = generate_votes(ongoing_validator_election_2, voters, ed25519_node_keys) + return votes + + +@pytest.fixture +def chain_migration_election_votes(b_mock, ongoing_chain_migration_election, ed25519_node_keys): + voters = ChainMigrationElection.recipients(b_mock) + votes = generate_votes(ongoing_chain_migration_election, voters, ed25519_node_keys) + return votes + + +@pytest.fixture +def chain_migration_election_votes_2(b_mock, ongoing_chain_migration_election_2, ed25519_node_keys): + voters = ChainMigrationElection.recipients(b_mock) + votes = generate_votes(ongoing_chain_migration_election_2, voters, ed25519_node_keys) + return votes + + +def generate_votes(election, voters, keys): + votes = [] + for voter in range(len(voters)): + v = gen_vote(election, voter, keys) + votes.append(v) + return votes diff --git a/tests/elections/__init__.py b/tests/elections/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/elections/test_election.py b/tests/elections/test_election.py new file mode 100644 index 00000000..0e9da188 --- /dev/null +++ b/tests/elections/test_election.py @@ -0,0 +1,64 @@ +from unittest.mock import MagicMock + +import pytest + +from bigchaindb.elections.election import Election + + +@pytest.mark.bdb +def test_approved_elections_one_migration_one_upsert( + b, + ongoing_validator_election, validator_election_votes, + ongoing_chain_migration_election, chain_migration_election_votes +): + txns = validator_election_votes + \ + chain_migration_election_votes + mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) + mock_chain_migration.assert_called_once() + mock_store_validator.assert_called_once() + + +@pytest.mark.bdb +def test_approved_elections_one_migration_two_upsert( + b, + ongoing_validator_election, validator_election_votes, + ongoing_validator_election_2, validator_election_votes_2, + ongoing_chain_migration_election, chain_migration_election_votes +): + txns = validator_election_votes + \ + validator_election_votes_2 + \ + chain_migration_election_votes + mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) + mock_chain_migration.assert_called_once() + mock_store_validator.assert_called_once() + + +@pytest.mark.bdb +def test_approved_elections_two_migrations_one_upsert( + b, + ongoing_validator_election, validator_election_votes, + ongoing_chain_migration_election, chain_migration_election_votes, + ongoing_chain_migration_election_2, chain_migration_election_votes_2 +): + txns = validator_election_votes + \ + chain_migration_election_votes + \ + chain_migration_election_votes_2 + mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) + assert mock_chain_migration.call_count == 2 + mock_store_validator.assert_called_once() + + +def test_approved_elections_no_elections(b): + txns = [] + mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) + mock_chain_migration.assert_not_called() + mock_store_validator.assert_not_called() + + +def run_approved_elections(bigchain, txns): + mock_chain_migration = MagicMock() + mock_store_validator = MagicMock() + bigchain.migrate_abci_chain = mock_chain_migration + bigchain.store_validator_set = mock_store_validator + Election.approved_elections(bigchain, 1, txns) + return mock_chain_migration, mock_store_validator diff --git a/tests/migrations/test_migration_election.py b/tests/migrations/test_migration_election.py new file mode 100644 index 00000000..b811903b --- /dev/null +++ b/tests/migrations/test_migration_election.py @@ -0,0 +1,9 @@ +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection + + +def test_valid_migration_election(b_mock, node_key): + voters = ChainMigrationElection.recipients(b_mock) + election = ChainMigrationElection.generate([node_key.public_key], + voters, + {}, None).sign([node_key.private_key]) + assert election.validate(b_mock) diff --git a/tests/upsert_validator/conftest.py b/tests/upsert_validator/conftest.py index 9ab2dad8..58b5e6dd 100644 --- a/tests/upsert_validator/conftest.py +++ b/tests/upsert_validator/conftest.py @@ -5,42 +5,12 @@ from unittest.mock import patch import pytest -from bigchaindb import Vote from bigchaindb.backend.localmongodb import query -from bigchaindb.lib import Block from bigchaindb.upsert_validator import ValidatorElection @pytest.fixture -def b_mock(b, network_validators): - b.get_validators = mock_get_validators(network_validators) - - return b - - -def mock_get_validators(network_validators): - def validator_set(height): - validators = [] - for public_key, power in network_validators.items(): - validators.append({ - 'public_key': {'type': 'ed25519-base64', 'value': public_key}, - 'voting_power': power - }) - return validators - - return validator_set - - -@pytest.fixture -def valid_election(b_mock, node_key, new_validator): - voters = ValidatorElection.recipients(b_mock) - return ValidatorElection.generate([node_key.public_key], - voters, - new_validator, None).sign([node_key.private_key]) - - -@pytest.fixture -def valid_election_b(b, node_key, new_validator): +def valid_upsert_validator_election_b(b, node_key, new_validator): voters = ValidatorElection.recipients(b) return ValidatorElection.generate([node_key.public_key], voters, @@ -57,30 +27,16 @@ def fixed_seed_election(b_mock, node_key, new_validator): @pytest.fixture -def ongoing_election(b, valid_election, ed25519_node_keys): - validators = b.get_validators(height=1) - genesis_validators = {'validators': validators, - 'height': 0, - 'election_id': None} - query.store_validator_set(b.connection, genesis_validators) - - b.store_bulk_transactions([valid_election]) - block_1 = Block(app_hash='hash_1', height=1, transactions=[valid_election.id]) - b.store_block(block_1._asdict()) - return valid_election - - -@pytest.fixture -def concluded_election(b, ongoing_election, ed25519_node_keys): +def concluded_election(b, ongoing_validator_election, ed25519_node_keys): election_result = {'height': 2, - 'election_id': ongoing_election.id} + 'election_id': ongoing_validator_election.id} query.store_election_results(b.connection, election_result) - return ongoing_election + return ongoing_validator_election @pytest.fixture -def inconclusive_election(b, ongoing_election, new_validator): +def inconclusive_election(b, ongoing_validator_election, new_validator): validators = b.get_validators(height=1) validators[0]['voting_power'] = 15 validator_update = {'validators': validators, @@ -88,20 +44,4 @@ def inconclusive_election(b, ongoing_election, new_validator): 'election_id': 'some_other_election'} query.store_validator_set(b.connection, validator_update) - return ongoing_election - - -def vote(election, voter, keys, b): - election_input = election.to_inputs()[voter] - votes = election.outputs[voter].amount - public_key = election_input.owners_before[0] - key = keys[public_key] - - election_pub_key = ValidatorElection.to_public_key(election.id) - - v = Vote.generate([election_input], - [([election_pub_key], votes)], - election_id=election.id)\ - .sign([key.private_key]) - b.store_bulk_transactions([v]) - return v + return ongoing_validator_election diff --git a/tests/upsert_validator/test_upsert_validator_vote.py b/tests/upsert_validator/test_upsert_validator_vote.py index b2e23af7..3c3512d0 100644 --- a/tests/upsert_validator/test_upsert_validator_vote.py +++ b/tests/upsert_validator/test_upsert_validator_vote.py @@ -5,104 +5,105 @@ import pytest import codecs +from bigchaindb.elections.election import Election from bigchaindb.tendermint_utils import public_key_to_base64 from bigchaindb.upsert_validator import ValidatorElection from bigchaindb.common.exceptions import AmountError from bigchaindb.common.crypto import generate_key_pair from bigchaindb.common.exceptions import ValidationError from bigchaindb.elections.vote import Vote -from tests.utils import generate_block +from tests.utils import generate_block, gen_vote pytestmark = [pytest.mark.execute] @pytest.mark.bdb -def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_node_keys): - b_mock.store_bulk_transactions([valid_election]) +def test_upsert_validator_valid_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + b_mock.store_bulk_transactions([valid_upsert_validator_election]) - input0 = valid_election.to_inputs()[0] - votes = valid_election.outputs[0].amount + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] - election_pub_key = ValidatorElection.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) vote = Vote.generate([input0], [([election_pub_key], votes)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) assert vote.validate(b_mock) @pytest.mark.bdb -def test_upsert_validator_valid_non_election_vote(b_mock, valid_election, ed25519_node_keys): - b_mock.store_bulk_transactions([valid_election]) +def test_upsert_validator_valid_non_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + b_mock.store_bulk_transactions([valid_upsert_validator_election]) - input0 = valid_election.to_inputs()[0] - votes = valid_election.outputs[0].amount + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] - election_pub_key = ValidatorElection.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) # Ensure that threshold conditions are now allowed with pytest.raises(ValidationError): Vote.generate([input0], [([election_pub_key, key0.public_key], votes)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) @pytest.mark.bdb -def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519_node_keys): +def test_upsert_validator_delegate_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): alice = generate_key_pair() - b_mock.store_bulk_transactions([valid_election]) + b_mock.store_bulk_transactions([valid_upsert_validator_election]) - input0 = valid_election.to_inputs()[0] - votes = valid_election.outputs[0].amount + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] delegate_vote = Vote.generate([input0], [([alice.public_key], 3), ([key0.public_key], votes-3)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) assert delegate_vote.validate(b_mock) b_mock.store_bulk_transactions([delegate_vote]) - election_pub_key = ValidatorElection.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) alice_votes = delegate_vote.to_inputs()[0] alice_casted_vote = Vote.generate([alice_votes], [([election_pub_key], 3)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([alice.private_key]) assert alice_casted_vote.validate(b_mock) key0_votes = delegate_vote.to_inputs()[1] key0_casted_vote = Vote.generate([key0_votes], [([election_pub_key], votes-3)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) assert key0_casted_vote.validate(b_mock) @pytest.mark.bdb -def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_node_keys): - b_mock.store_bulk_transactions([valid_election]) +def test_upsert_validator_invalid_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + b_mock.store_bulk_transactions([valid_upsert_validator_election]) - input0 = valid_election.to_inputs()[0] - votes = valid_election.outputs[0].amount + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] - election_pub_key = ValidatorElection.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) vote = Vote.generate([input0], [([election_pub_key], votes+1)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) with pytest.raises(AmountError): @@ -110,113 +111,111 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_ @pytest.mark.bdb -def test_valid_election_votes_received(b_mock, valid_election, ed25519_node_keys): +def test_valid_election_votes_received(b_mock, valid_upsert_validator_election, ed25519_node_keys): alice = generate_key_pair() - b_mock.store_bulk_transactions([valid_election]) - assert valid_election.get_commited_votes(b_mock) == 0 + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + assert valid_upsert_validator_election.get_commited_votes(b_mock) == 0 - input0 = valid_election.to_inputs()[0] - votes = valid_election.outputs[0].amount + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] # delegate some votes to alice delegate_vote = Vote.generate([input0], [([alice.public_key], 4), ([key0.public_key], votes-4)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) b_mock.store_bulk_transactions([delegate_vote]) - assert valid_election.get_commited_votes(b_mock) == 0 + assert valid_upsert_validator_election.get_commited_votes(b_mock) == 0 - election_public_key = ValidatorElection.to_public_key(valid_election.id) + election_public_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) alice_votes = delegate_vote.to_inputs()[0] key0_votes = delegate_vote.to_inputs()[1] alice_casted_vote = Vote.generate([alice_votes], [([election_public_key], 2), ([alice.public_key], 2)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([alice.private_key]) assert alice_casted_vote.validate(b_mock) b_mock.store_bulk_transactions([alice_casted_vote]) # Check if the delegated vote is count as valid vote - assert valid_election.get_commited_votes(b_mock) == 2 + assert valid_upsert_validator_election.get_commited_votes(b_mock) == 2 key0_casted_vote = Vote.generate([key0_votes], [([election_public_key], votes-4)], - election_id=valid_election.id)\ + election_id=valid_upsert_validator_election.id)\ .sign([key0.private_key]) assert key0_casted_vote.validate(b_mock) b_mock.store_bulk_transactions([key0_casted_vote]) - assert valid_election.get_commited_votes(b_mock) == votes-2 + assert valid_upsert_validator_election.get_commited_votes(b_mock) == votes - 2 @pytest.mark.bdb -def test_valid_election_conclude(b_mock, valid_election, ed25519_node_keys): +def test_valid_election_conclude(b_mock, valid_upsert_validator_election, ed25519_node_keys): # Node 0: cast vote - tx_vote0 = gen_vote(valid_election, 0, ed25519_node_keys) + tx_vote0 = gen_vote(valid_upsert_validator_election, 0, ed25519_node_keys) # check if the vote is valid even before the election doesn't exist with pytest.raises(ValidationError): assert tx_vote0.validate(b_mock) # store election - b_mock.store_bulk_transactions([valid_election]) + b_mock.store_bulk_transactions([valid_upsert_validator_election]) # cannot conclude election as not votes exist - assert not ValidatorElection.has_concluded(b_mock, valid_election.id) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) # validate vote assert tx_vote0.validate(b_mock) - assert not ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote0]) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote0]) b_mock.store_bulk_transactions([tx_vote0]) - assert not ValidatorElection.has_concluded(b_mock, valid_election.id) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) # Node 1: cast vote - tx_vote1 = gen_vote(valid_election, 1, ed25519_node_keys) + tx_vote1 = gen_vote(valid_upsert_validator_election, 1, ed25519_node_keys) # Node 2: cast vote - tx_vote2 = gen_vote(valid_election, 2, ed25519_node_keys) + tx_vote2 = gen_vote(valid_upsert_validator_election, 2, ed25519_node_keys) # Node 3: cast vote - tx_vote3 = gen_vote(valid_election, 3, ed25519_node_keys) + tx_vote3 = gen_vote(valid_upsert_validator_election, 3, ed25519_node_keys) assert tx_vote1.validate(b_mock) - assert not ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote1]) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote1]) # 2/3 is achieved in the same block so the election can be.has_concludedd - assert ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote1, tx_vote2]) + assert ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote1, tx_vote2]) b_mock.store_bulk_transactions([tx_vote1]) - assert not ValidatorElection.has_concluded(b_mock, valid_election.id) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) assert tx_vote2.validate(b_mock) assert tx_vote3.validate(b_mock) # conclusion can be triggered my different votes in the same block - assert ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote2]) - assert ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote2, tx_vote3]) + assert ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote2]) + assert ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote2, tx_vote3]) b_mock.store_bulk_transactions([tx_vote2]) # Once the blockchain records >2/3 of the votes the election is assumed to be.has_concludedd # so any invocation of `.has_concluded` for that election should return False - assert not ValidatorElection.has_concluded(b_mock, valid_election.id) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) # Vote is still valid but the election cannot be.has_concludedd as it it assmed that it has # been.has_concludedd before assert tx_vote3.validate(b_mock) - assert not ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote3]) + assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote3]) @pytest.mark.abci def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys): - import time - import requests if b.get_latest_block()['height'] == 0: generate_block(b) @@ -244,20 +243,17 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys): new_validator, None).sign([node_key.private_key]) code, message = b.write_transaction(election, 'broadcast_tx_commit') assert code == 202 - time.sleep(3) - assert b.get_transaction(election.id) tx_vote = gen_vote(election, 0, ed25519_node_keys) assert tx_vote.validate(b) code, message = b.write_transaction(tx_vote, 'broadcast_tx_commit') assert code == 202 - time.sleep(3) - resp = requests.get(b.endpoint + 'validators') + resp = b.get_validators() validator_pub_keys = [] - for v in resp.json()['result']['validators']: - validator_pub_keys.append(v['pub_key']['value']) + for v in resp: + validator_pub_keys.append(v['public_key']['value']) assert (public_key64 in validator_pub_keys) new_validator_set = b.get_validators() @@ -293,19 +289,19 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): assert not ValidatorElection.has_concluded(b, election.id, [tx_vote0, tx_vote1]) assert ValidatorElection.has_concluded(b, election.id, [tx_vote0, tx_vote1, tx_vote2]) - assert not ValidatorElection.approved_update(b, 4, [tx_vote0]) - assert not ValidatorElection.approved_update(b, 4, [tx_vote0, tx_vote1]) + assert Election.approved_elections(b, 4, [tx_vote0]) == [] + assert Election.approved_elections(b, 4, [tx_vote0, tx_vote1]) == [] - update = ValidatorElection.approved_update(b, 4, [tx_vote0, tx_vote1, tx_vote2]) - assert update - update_public_key = codecs.encode(update.pub_key.data, 'base64').decode().rstrip('\n') + update = Election.approved_elections(b, 4, [tx_vote0, tx_vote1, tx_vote2]) + assert len(update) == 1 + update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') assert update_public_key == public_key64 b.store_bulk_transactions([tx_vote0, tx_vote1]) - update = ValidatorElection.approved_update(b, 4, [tx_vote2]) - assert update - update_public_key = codecs.encode(update.pub_key.data, 'base64').decode().rstrip('\n') + update = Election.approved_elections(b, 4, [tx_vote2]) + assert len(update) == 1 + update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') assert update_public_key == public_key64 # remove validator @@ -326,10 +322,9 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): b.store_bulk_transactions([tx_vote0, tx_vote1]) - update = ValidatorElection.approved_update(b, 9, [tx_vote2]) - if update: - update_public_key = codecs.encode(update.pub_key.data, 'base64').decode().rstrip('\n') - assert update + update = Election.approved_elections(b, 9, [tx_vote2]) + assert len(update) == 1 + update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') assert update_public_key == public_key64 # assert that the public key is not a part of the current validator set @@ -340,22 +335,6 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): # ============================================================================ # Helper functions # ============================================================================ -def to_inputs(election, i, ed25519_node_keys): - input0 = election.to_inputs()[i] - votes = election.outputs[i].amount - public_key0 = input0.owners_before[0] - key0 = ed25519_node_keys[public_key0] - return (input0, votes, key0) - - -def gen_vote(election, i, ed25519_node_keys): - (input_i, votes_i, key_i) = to_inputs(election, i, ed25519_node_keys) - election_pub_key = ValidatorElection.to_public_key(election.id) - return Vote.generate([input_i], - [([election_pub_key], votes_i)], - election_id=election.id)\ - .sign([key_i.private_key]) - def reset_validator_set(b, node_keys, height): validators = [] diff --git a/tests/upsert_validator/test_validator_election.py b/tests/upsert_validator/test_validator_election.py index c1f56fff..205be554 100644 --- a/tests/upsert_validator/test_validator_election.py +++ b/tests/upsert_validator/test_validator_election.py @@ -111,9 +111,9 @@ def test_upsert_validator_invalid_election(b_mock, new_validator, node_key, fixe tx_election.validate(b_mock) -def test_get_status_ongoing(b, ongoing_election, new_validator): +def test_get_status_ongoing(b, ongoing_validator_election, new_validator): status = ValidatorElection.ONGOING - resp = ongoing_election.get_status(b) + resp = ongoing_validator_election.get_status(b) assert resp == status @@ -124,6 +124,9 @@ def test_get_status_concluded(b, concluded_election, new_validator): def test_get_status_inconclusive(b, inconclusive_election, new_validator): + def set_block_height_to_3(): + return {'height': 3} + def custom_mock_get_validators(height): if height >= 3: return [{'pub_key': {'data': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=', @@ -153,18 +156,19 @@ def test_get_status_inconclusive(b, inconclusive_election, new_validator): 'voting_power': 8}] b.get_validators = custom_mock_get_validators + b.get_latest_block = set_block_height_to_3 status = ValidatorElection.INCONCLUSIVE resp = inconclusive_election.get_status(b) assert resp == status -def test_upsert_validator_show(caplog, ongoing_election, b): +def test_upsert_validator_show(caplog, ongoing_validator_election, b): from bigchaindb.commands.bigchaindb import run_election_show - election_id = ongoing_election.id - public_key = public_key_to_base64(ongoing_election.asset['data']['public_key']['value']) - power = ongoing_election.asset['data']['power'] - node_id = ongoing_election.asset['data']['node_id'] + election_id = ongoing_validator_election.id + public_key = public_key_to_base64(ongoing_validator_election.asset['data']['public_key']['value']) + power = ongoing_validator_election.asset['data']['power'] + node_id = ongoing_validator_election.asset['data']['node_id'] status = ValidatorElection.ONGOING show_args = Namespace(action='show', diff --git a/tests/utils.py b/tests/utils.py index 87c07a79..852be5e6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,8 +4,10 @@ from functools import singledispatch +from bigchaindb import Vote from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection from bigchaindb.backend.schema import TABLES +from bigchaindb.elections.election import Election @singledispatch @@ -33,3 +35,20 @@ def generate_block(bigchain): code, message = bigchain.write_transaction(tx, 'broadcast_tx_commit') assert code == 202 time.sleep(2) + + +def to_inputs(election, i, ed25519_node_keys): + input0 = election.to_inputs()[i] + votes = election.outputs[i].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + return (input0, votes, key0) + + +def gen_vote(election, i, ed25519_node_keys): + (input_i, votes_i, key_i) = to_inputs(election, i, ed25519_node_keys) + election_pub_key = Election.to_public_key(election.id) + return Vote.generate([input_i], + [([election_pub_key], votes_i)], + election_id=election.id)\ + .sign([key_i.private_key]) From 8e5f77070712dd83cd0e2c5fb582ac0bf0e524d3 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 17 Sep 2018 14:06:48 +0200 Subject: [PATCH 02/13] List the "blocks" endpoint in response to HTTP info requests (#2545) * Problem: HTTP info endpoints don't include "blocks" Solution: Add "blocks" to the response to the HTTP info endpoints * Removed "blocks" endpoints from "Advanced Usage" because they are really not that advanced --- bigchaindb/web/views/info.py | 1 + docs/server/source/http-client-server-api.rst | 8 +------- tests/web/test_info.py | 2 ++ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index 4b44157b..fbbb0392 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -47,6 +47,7 @@ def get_api_v1_info(api_prefix): return { 'docs': ''.join(docs_url), 'transactions': '{}transactions/'.format(api_prefix), + 'blocks': '{}blocks/'.format(api_prefix), 'assets': '{}assets/'.format(api_prefix), 'outputs': '{}outputs/'.format(api_prefix), 'streams': websocket_root, diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index eadfe983..325320f1 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -616,14 +616,8 @@ Validators :statuscode 200: The query was executed successfully and validators set was returned. -Advanced Usage --------------------------------- - -The following endpoints are more advanced -and meant for debugging and transparency purposes. - Blocks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------ .. http:get:: /api/v1/blocks/{block_height} diff --git a/tests/web/test_info.py b/tests/web/test_info.py index b91b6f60..41034e2f 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -16,6 +16,7 @@ def test_api_root_endpoint(client, wsserver_base_url): 'v1': { 'docs': ''.join(docs_url), 'transactions': '/api/v1/transactions/', + 'blocks': '/api/v1/blocks/', 'assets': '/api/v1/assets/', 'outputs': '/api/v1/outputs/', 'streams': '{}/api/v1/streams/valid_transactions'.format( @@ -38,6 +39,7 @@ def test_api_v1_endpoint(client, wsserver_base_url): api_v1_info = { 'docs': ''.join(docs_url), 'transactions': '/transactions/', + 'blocks': '/blocks/', 'assets': '/assets/', 'outputs': '/outputs/', 'streams': '{}/api/v1/streams/valid_transactions'.format( From e3e95836d076719488d5d6bf8f8148ff8a01ece8 Mon Sep 17 00:00:00 2001 From: Muawia Khan Date: Mon, 17 Sep 2018 14:10:22 +0200 Subject: [PATCH 03/13] Problem: bigchaindb not starting because of a module dependency (#2548) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a004178d..4181bddf 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ install_requires = [ 'aiohttp~=3.0', 'bigchaindb-abci==0.5.1', 'setproctitle~=1.1.0', + 'packaging~=17.0', ] setup( From 16355748ddda8e18b5691c7d31249c5e4aca559d Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Mon, 17 Sep 2018 14:39:01 +0200 Subject: [PATCH 04/13] Problem: Docs build fails (#2549) Solution: Create __init__.py so that migrations can be imported properly --- bigchaindb/migrations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 bigchaindb/migrations/__init__.py diff --git a/bigchaindb/migrations/__init__.py b/bigchaindb/migrations/__init__.py new file mode 100644 index 00000000..e69de29b From eb0e387b18af977840e2c12cf22f230be4b94ad9 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 17 Sep 2018 14:43:03 +0200 Subject: [PATCH 05/13] Problem: CHANGELOG.md hasn't been updated since Aug. 1 (#2509) Last update to CHANGELOG.md before 2.0.0b6 --- CHANGELOG.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8f3201a..d84acdd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,60 @@ For reference, the possible headings are: * **Known Issues** * **Notes** +## [2.0 Beta 6] - 2018-09-17 + +Tag name: v2.0.0b6 + +### Added + +* [New documentation about privacy and handling private data](https://docs.bigchaindb.com/en/latest/private-data.html). [Pull request #2437](https://github.com/bigchaindb/bigchaindb/pull/2437) +* New documentation about log rotation. Also rotate Tendermint logs if started using Monit. [Pull request #2528](https://github.com/bigchaindb/bigchaindb/pull/2528) +* Began implementing one of the migration strategies outlined in [BEP-42](https://github.com/bigchaindb/BEPs/tree/master/42). That involved creating a more general-purpose election process and commands. Pull requests [#2488](https://github.com/bigchaindb/bigchaindb/pull/2488), [#2495](https://github.com/bigchaindb/bigchaindb/pull/2495), [#2498](https://github.com/bigchaindb/bigchaindb/pull/2498), [#2515](https://github.com/bigchaindb/bigchaindb/pull/2515), [#2535](https://github.com/bigchaindb/bigchaindb/pull/2535) +* Used memoization to avoid doing some validation checks multiple times. [Pull request #2490](https://github.com/bigchaindb/bigchaindb/pull/2490) +* Created an all-in-one Docker image containing BigchainDB Server, Tendermint and MongoDB. It was created for a particular user and is not recommended for production use unless you really know what you're doing. [Pull request #2424](https://github.com/bigchaindb/bigchaindb/pull/2424) + +### Changed + +* The supported versions of Tendermint are now hard-wired into BigchainDB Server: it checks to see what version the connected Tendermint has, and if it's not compatible, BigchainDB Server exits with an error message. [Pull request #2541](https://github.com/bigchaindb/bigchaindb/pull/2541) +* The docs no longer say to install the highest version of Tendermint: they say to install a specific version. [Pull request #2524](https://github.com/bigchaindb/bigchaindb/pull/2524) +* The setup docs include more recommended settings for `config.toml`. [Pull request #2516](https://github.com/bigchaindb/bigchaindb/pull/2516) +* The process to add, remove or update the voting power of a validator at run time (using the `bigchaindb upsert-validator` subcommands) was completely changed and is now fully working. See [issue #2372](https://github.com/bigchaindb/bigchaindb/issues/2372) and all the pull requests it references. Pull requests [#2439](https://github.com/bigchaindb/bigchaindb/pull/2439) and [#2440](https://github.com/bigchaindb/bigchaindb/pull/2440) +* The license on the documentation was changed from CC-BY-SA-4 to CC-BY-4. [Pull request #2427](https://github.com/bigchaindb/bigchaindb/pull/2427) +* Re-activated and/or updated some unit tests that had been deacivated during the migration to Tendermint. Pull requests [#2390](https://github.com/bigchaindb/bigchaindb/pull/2390), [#2415](https://github.com/bigchaindb/bigchaindb/pull/2415), [#2452](https://github.com/bigchaindb/bigchaindb/pull/24), [#2456](https://github.com/bigchaindb/bigchaindb/pull/2456) +* Updated RapidJSON to a newer, faster version. [Pull request #2470](https://github.com/bigchaindb/bigchaindb/pull/2470) +* The Java driver is now officially supported. [Pull request #2478](https://github.com/bigchaindb/bigchaindb/pull/2478) +* The MongoDB indexes on transaction id and block height were changed to be [unique indexes](https://docs.mongodb.com/manual/core/index-unique/). [Pull request #2492](https://github.com/bigchaindb/bigchaindb/pull/2492) +* Updated the required `cryptoconditions` package to a newer one. [Pull request #2494](https://github.com/bigchaindb/bigchaindb/pull/2494) + +### Removed + +* Removed some old code and tests. Pull requests + [#2374](https://github.com/bigchaindb/bigchaindb/pull/2374), + [#2452](https://github.com/bigchaindb/bigchaindb/pull/2452), + [#2474](https://github.com/bigchaindb/bigchaindb/pull/2474), + [#2476](https://github.com/bigchaindb/bigchaindb/pull/2476), + [#2491](https://github.com/bigchaindb/bigchaindb/pull/2491) + +### Fixed + +* Fixed the Events API so that it only sends valid transactions to subscribers. Also changed how it works internally, so now it is more reliable. [Pull request #2529](https://github.com/bigchaindb/bigchaindb/pull/2529) +* Fixed a bug where MongoDB database initialization would abort if a collection already existed. [Pull request #2520](https://github.com/bigchaindb/bigchaindb/pull/2520) +* Fixed a unit test that was failing randomly. [Pull request #2423](https://github.com/bigchaindb/bigchaindb/pull/2423) +* Fixed the validator curl port. [Pull request #2447](https://github.com/bigchaindb/bigchaindb/pull/2447) +* Fixed an error in the docs about the HTTP POST /transactions endpoint. [Pull request #2481](https://github.com/bigchaindb/bigchaindb/pull/2481) +* Fixed a unit test that could loop forever. [Pull requqest #2486](https://github.com/bigchaindb/bigchaindb/pull/2486) +* Fixed a bug when validating a CREATE + TRANSFER. [Pull request #2487](https://github.com/bigchaindb/bigchaindb/pull/2487) +* Fixed the HTTP response when posting a transaction in commit mode. [Pull request #2510](https://github.com/bigchaindb/bigchaindb/pull/2510) +* Fixed a crash that happened when attempting to restart BigchainDB at Tendermint block height 1. [Pull request#2519](https://github.com/bigchaindb/bigchaindb/pull/2519) + +### External Contributors + +@danacr - [Pull request #2447](https://github.com/bigchaindb/bigchaindb/pull/2447) + +### Notes + +The docs section titled "Production Deployment Template" was renamed to "Kubernetes Deployment Template" and we no longer consider it the go-to deployment template. The "Simple Deployment Template" is simpler, easier to understand, and less expensive (unless you are with an organization that already has a big Kubernetes cluster). + ## [2.0 Beta 5] - 2018-08-01 Tag name: v2.0.0b5 From 528ba07c68f5ef64ad63b1ddaecca3171d55c7e4 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 17 Sep 2018 15:00:26 +0200 Subject: [PATCH 06/13] Last PR before 2.0.0b6 (#2550) * Update k8s files for 2.0.0b6 * Update version.py for 2.0.0b6 --- bigchaindb/version.py | 4 ++-- k8s/bigchaindb/bigchaindb-ss.yaml | 2 +- k8s/dev-setup/bigchaindb.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 277f5645..b1761387 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) # Code is Apache-2.0 and docs are CC-BY-4.0 -__version__ = '2.0.0b5' -__short_version__ = '2.0b5' +__version__ = '2.0.0b6' +__short_version__ = '2.0b6' # supported Tendermint version __tm_supported_versions__ = ["0.22.8"] diff --git a/k8s/bigchaindb/bigchaindb-ss.yaml b/k8s/bigchaindb/bigchaindb-ss.yaml index 1371f884..b6ada5a3 100644 --- a/k8s/bigchaindb/bigchaindb-ss.yaml +++ b/k8s/bigchaindb/bigchaindb-ss.yaml @@ -158,7 +158,7 @@ spec: timeoutSeconds: 15 # BigchainDB container - name: bigchaindb - image: bigchaindb/bigchaindb:2.0.0-beta5 + image: bigchaindb/bigchaindb:2.0.0-beta6 imagePullPolicy: Always args: - start diff --git a/k8s/dev-setup/bigchaindb.yaml b/k8s/dev-setup/bigchaindb.yaml index f1d19615..7a2fa34c 100644 --- a/k8s/dev-setup/bigchaindb.yaml +++ b/k8s/dev-setup/bigchaindb.yaml @@ -38,7 +38,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:2.0.0-beta5 + image: bigchaindb/bigchaindb:2.0.0-beta6 imagePullPolicy: Always args: - start From cf6fa6b5539610df21e18700d0c2fa676d21486a Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Mon, 17 Sep 2018 15:36:38 +0200 Subject: [PATCH 07/13] Problem: Election type not casted for runnign command (#2551) Solution: Replace '-' with '_' before running election sub-command --- bigchaindb/commands/bigchaindb.py | 3 ++- tests/commands/test_commands.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index f9588dcd..58db7ac0 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -113,7 +113,8 @@ def run_election(args): def run_election_new(args, bigchain): - globals()[f'run_election_new_{args.election_type}'](args, bigchain) + election_type = args.election_type.replace('-', '_') + globals()[f'run_election_new_{election_type}'](args, bigchain) def create_new_election(sk, bigchain, election_class, data): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index bbeea227..f9b14f89 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -306,7 +306,7 @@ def test_election_new_upsert_validator_with_tendermint(b, priv_validator_path, u from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator new_args = Namespace(action='new', - election_type='upsert_validator', + election_type='upsert-validator', public_key='HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52GnGuPY=', power=1, node_id='unique_node_id_for_test_upsert_validator_new_with_tendermint', @@ -330,7 +330,7 @@ def test_election_new_upsert_validator_without_tendermint(caplog, b, priv_valida b.write_transaction = mock_write args = Namespace(action='new', - election_type='upsert_validator', + election_type='upsert-validator', public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', power=1, node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', @@ -384,7 +384,7 @@ def test_election_new_upsert_validator_invalid_election(caplog, b, priv_validato from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator args = Namespace(action='new', - election_type='upsert_validator', + election_type='upsert-validator', public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', power=10, node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', @@ -408,7 +408,7 @@ def test_election_new_upsert_validator_invalid_power(caplog, b, priv_validator_p b.write_transaction = mock_write b.get_validators = mock_get_validators args = Namespace(action='new', - election_type='upsert_validator', + election_type='upsert-validator', public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', power=10, node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', @@ -427,7 +427,7 @@ def test_election_approve_with_tendermint(b, priv_validator_path, user_sk, valid public_key = 'CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=' new_args = Namespace(action='new', - election_type='upsert_validator', + election_type='upsert-validator', public_key=public_key, power=1, node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', From 126e90e7321f571dfbc7bc25b165b21ee6ebc9d1 Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Tue, 18 Sep 2018 12:32:16 +0200 Subject: [PATCH 08/13] Adjust multiple elections conclusion. (#2553) - Do not conclude migration election if there is a migration in progress. - Rewrite election tests to not use mocks and assert many different things. - Record concluded elections in the `election` collection. --- bigchaindb/backend/localmongodb/query.py | 5 +- bigchaindb/elections/election.py | 42 ++-- .../migrations/chain_migration_election.py | 9 + tests/conftest.py | 51 +---- tests/elections/test_election.py | 179 +++++++++++++----- .../test_upsert_validator_vote.py | 33 ++-- tests/utils.py | 64 ++++++- 7 files changed, 236 insertions(+), 147 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 75fa7aa5..6b0793ba 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -283,12 +283,11 @@ def store_validator_set(conn, validators_update): @register_query(LocalMongoDBConnection) def store_election_results(conn, election): - height = election['height'] return conn.run( conn.collection('elections').replace_one( - {'height': height}, + {'election_id': election['election_id']}, election, - upsert=True + upsert=True, ) ) diff --git a/bigchaindb/elections/election.py b/bigchaindb/elections/election.py index ca0a4be3..40861a52 100644 --- a/bigchaindb/elections/election.py +++ b/bigchaindb/elections/election.py @@ -186,27 +186,26 @@ class Election(Transaction): election_pk)) return self.count_votes(election_pk, txns, dict.get) - @classmethod - def has_concluded(cls, bigchain, election_id, current_votes=[], height=None): - """Check if the given `election_id` can be concluded or not - NOTE: - * Election is concluded iff the current validator set is exactly equal - to the validator set encoded in election outputs - * Election can concluded only if the current votes achieves a supermajority + def has_concluded(self, bigchain, current_votes=[], height=None): + """Check if the election can be concluded or not. + + * Elections can only be concluded if the current validator set + is exactly equal to the validator set encoded in the election outputs. + * Elections can be concluded only if the current votes form a supermajority. + + Custom elections may override this function and introduce additional checks. """ - election = bigchain.get_transaction(election_id) - if election: - election_pk = election.to_public_key(election.id) - votes_committed = election.get_commited_votes(bigchain, election_pk) - votes_current = election.count_votes(election_pk, current_votes) - current_validators = election.get_validators(bigchain, height) + election_pk = self.to_public_key(self.id) + votes_committed = self.get_commited_votes(bigchain, election_pk) + votes_current = self.count_votes(election_pk, current_votes) + current_validators = self.get_validators(bigchain, height) - if election.is_same_topology(current_validators, election.outputs): - total_votes = sum(current_validators.values()) - if (votes_committed < (2/3)*total_votes) and \ - (votes_committed + votes_current >= (2/3)*total_votes): - return election + if self.is_same_topology(current_validators, self.outputs): + total_votes = sum(current_validators.values()) + if (votes_committed < (2/3) * total_votes) and \ + (votes_committed + votes_current >= (2/3)*total_votes): + return True return False def get_status(self, bigchain): @@ -255,9 +254,11 @@ class Election(Transaction): validator_set_updated = False validator_set_change = [] for election_id, votes in elections.items(): - election = Election.has_concluded(bigchain, election_id, votes, new_height) + election = bigchain.get_transaction(election_id) + if election is None: + continue - if not election: + if not election.has_concluded(bigchain, votes, new_height): continue if election.makes_validator_set_change(): @@ -267,6 +268,7 @@ class Election(Transaction): validator_set_updated = True election.on_approval(bigchain, election, new_height) + election.store_election_results(bigchain, election, new_height) return validator_set_change diff --git a/bigchaindb/migrations/chain_migration_election.py b/bigchaindb/migrations/chain_migration_election.py index e1e783d0..8e80b979 100644 --- a/bigchaindb/migrations/chain_migration_election.py +++ b/bigchaindb/migrations/chain_migration_election.py @@ -10,6 +10,15 @@ class ChainMigrationElection(Election): TX_SCHEMA_CUSTOM = TX_SCHEMA_CHAIN_MIGRATION_ELECTION CHANGES_VALIDATOR_SET = False + def has_concluded(self, bigchaindb, *args, **kwargs): + chain = bigchaindb.get_latest_abci_chain() + if chain is not None and not chain['is_synced']: + # do not conclude the migration election if + # there is another migration in progress + return False + + return super().has_concluded(bigchaindb, *args, **kwargs) + @classmethod def on_approval(cls, bigchain, election, new_height): bigchain.migrate_abci_chain() diff --git a/tests/conftest.py b/tests/conftest.py index 4c271aad..90d54dea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,6 @@ from pymongo import MongoClient from bigchaindb import ValidatorElection from bigchaindb.common import crypto from bigchaindb.log import setup_logging -from bigchaindb.migrations.chain_migration_election import ChainMigrationElection from bigchaindb.tendermint_utils import key_from_base64 from bigchaindb.backend import schema, query from bigchaindb.common.crypto import (key_pair_from_ed25519_key, @@ -714,22 +713,6 @@ def valid_upsert_validator_election_2(b_mock, node_key, new_validator): new_validator, None).sign([node_key.private_key]) -@pytest.fixture -def valid_chain_migration_election(b_mock, node_key): - voters = ChainMigrationElection.recipients(b_mock) - return ChainMigrationElection.generate([node_key.public_key], - voters, - {}, None).sign([node_key.private_key]) - - -@pytest.fixture -def valid_chain_migration_election_2(b_mock, node_key): - voters = ChainMigrationElection.recipients(b_mock) - return ChainMigrationElection.generate([node_key.public_key], - voters, - {}, None).sign([node_key.private_key]) - - @pytest.fixture def ongoing_validator_election(b, valid_upsert_validator_election, ed25519_node_keys): validators = b.get_validators(height=1) @@ -758,24 +741,6 @@ def ongoing_validator_election_2(b, valid_upsert_validator_election_2, ed25519_n return valid_upsert_validator_election_2 -@pytest.fixture -def ongoing_chain_migration_election(b, valid_chain_migration_election, ed25519_node_keys): - - b.store_bulk_transactions([valid_chain_migration_election]) - block_1 = Block(app_hash='hash_1', height=1, transactions=[valid_chain_migration_election.id]) - b.store_block(block_1._asdict()) - return valid_chain_migration_election - - -@pytest.fixture -def ongoing_chain_migration_election_2(b, valid_chain_migration_election_2, ed25519_node_keys): - - b.store_bulk_transactions([valid_chain_migration_election_2]) - block_1 = Block(app_hash='hash_2', height=1, transactions=[valid_chain_migration_election_2.id]) - b.store_block(block_1._asdict()) - return valid_chain_migration_election_2 - - @pytest.fixture def validator_election_votes(b_mock, ongoing_validator_election, ed25519_node_keys): voters = ValidatorElection.recipients(b_mock) @@ -790,23 +755,9 @@ def validator_election_votes_2(b_mock, ongoing_validator_election_2, ed25519_nod return votes -@pytest.fixture -def chain_migration_election_votes(b_mock, ongoing_chain_migration_election, ed25519_node_keys): - voters = ChainMigrationElection.recipients(b_mock) - votes = generate_votes(ongoing_chain_migration_election, voters, ed25519_node_keys) - return votes - - -@pytest.fixture -def chain_migration_election_votes_2(b_mock, ongoing_chain_migration_election_2, ed25519_node_keys): - voters = ChainMigrationElection.recipients(b_mock) - votes = generate_votes(ongoing_chain_migration_election_2, voters, ed25519_node_keys) - return votes - - def generate_votes(election, voters, keys): votes = [] - for voter in range(len(voters)): + for voter, _ in enumerate(voters): v = gen_vote(election, voter, keys) votes.append(v) return votes diff --git a/tests/elections/test_election.py b/tests/elections/test_election.py index 0e9da188..0aed1816 100644 --- a/tests/elections/test_election.py +++ b/tests/elections/test_election.py @@ -1,64 +1,143 @@ -from unittest.mock import MagicMock - import pytest +from tests.utils import generate_election, generate_validators + +from bigchaindb.lib import Block from bigchaindb.elections.election import Election +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection +from bigchaindb.upsert_validator.validator_election import ValidatorElection @pytest.mark.bdb -def test_approved_elections_one_migration_one_upsert( - b, - ongoing_validator_election, validator_election_votes, - ongoing_chain_migration_election, chain_migration_election_votes -): - txns = validator_election_votes + \ - chain_migration_election_votes - mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) - mock_chain_migration.assert_called_once() - mock_store_validator.assert_called_once() +def test_approved_elections_concludes_all_elections(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election']) + txs = [election] + total_votes = votes + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}) + + txs += [election] + total_votes += votes + + b.store_abci_chain(1, 'chain-X') + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.approved_elections(b, 1, total_votes) + + validators = b.get_validators() + assert len(validators) == 5 + assert new_validator['storage'] in validators + + chain = b.get_latest_abci_chain() + assert chain + assert chain == { + 'height': 2, + 'is_synced': False, + 'chain_id': 'chain-X-migrated-at-height-1', + } + + for tx in txs: + election = b.get_election(tx.id) + assert election @pytest.mark.bdb -def test_approved_elections_one_migration_two_upsert( - b, - ongoing_validator_election, validator_election_votes, - ongoing_validator_election_2, validator_election_votes_2, - ongoing_chain_migration_election, chain_migration_election_votes -): - txns = validator_election_votes + \ - validator_election_votes_2 + \ - chain_migration_election_votes - mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) - mock_chain_migration.assert_called_once() - mock_store_validator.assert_called_once() +def test_approved_elections_applies_only_one_validator_update(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election']) + txs = [election] + total_votes = votes + + another_validator = generate_validators([1])[0] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + another_validator['election']) + txs += [election] + total_votes += votes + + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.approved_elections(b, 1, total_votes) + + validators = b.get_validators() + assert len(validators) == 5 + assert new_validator['storage'] in validators + assert another_validator['storage'] not in validators + + assert b.get_election(txs[0].id) + assert not b.get_election(txs[1].id) @pytest.mark.bdb -def test_approved_elections_two_migrations_one_upsert( - b, - ongoing_validator_election, validator_election_votes, - ongoing_chain_migration_election, chain_migration_election_votes, - ongoing_chain_migration_election_2, chain_migration_election_votes_2 -): - txns = validator_election_votes + \ - chain_migration_election_votes + \ - chain_migration_election_votes_2 - mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) - assert mock_chain_migration.call_count == 2 - mock_store_validator.assert_called_once() +def test_approved_elections_applies_only_one_migration(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}) + txs = [election] + total_votes = votes + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}) + + txs += [election] + total_votes += votes + + b.store_abci_chain(1, 'chain-X') + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.approved_elections(b, 1, total_votes) + chain = b.get_latest_abci_chain() + assert chain + assert chain == { + 'height': 2, + 'is_synced': False, + 'chain_id': 'chain-X-migrated-at-height-1', + } + + assert b.get_election(txs[0].id) + assert not b.get_election(txs[1].id) -def test_approved_elections_no_elections(b): - txns = [] - mock_chain_migration, mock_store_validator = run_approved_elections(b, txns) - mock_chain_migration.assert_not_called() - mock_store_validator.assert_not_called() - - -def run_approved_elections(bigchain, txns): - mock_chain_migration = MagicMock() - mock_store_validator = MagicMock() - bigchain.migrate_abci_chain = mock_chain_migration - bigchain.store_validator_set = mock_store_validator - Election.approved_elections(bigchain, 1, txns) - return mock_chain_migration, mock_store_validator +def test_approved_elections_gracefully_handles_empty_block(b): + Election.approved_elections(b, 1, []) diff --git a/tests/upsert_validator/test_upsert_validator_vote.py b/tests/upsert_validator/test_upsert_validator_vote.py index 3c3512d0..7c49bd77 100644 --- a/tests/upsert_validator/test_upsert_validator_vote.py +++ b/tests/upsert_validator/test_upsert_validator_vote.py @@ -168,14 +168,14 @@ def test_valid_election_conclude(b_mock, valid_upsert_validator_election, ed2551 # store election b_mock.store_bulk_transactions([valid_upsert_validator_election]) # cannot conclude election as not votes exist - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) + assert not valid_upsert_validator_election.has_concluded(b_mock) # validate vote assert tx_vote0.validate(b_mock) - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote0]) + assert not valid_upsert_validator_election.has_concluded(b_mock, [tx_vote0]) b_mock.store_bulk_transactions([tx_vote0]) - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) + assert not valid_upsert_validator_election.has_concluded(b_mock) # Node 1: cast vote tx_vote1 = gen_vote(valid_upsert_validator_election, 1, ed25519_node_keys) @@ -187,31 +187,31 @@ def test_valid_election_conclude(b_mock, valid_upsert_validator_election, ed2551 tx_vote3 = gen_vote(valid_upsert_validator_election, 3, ed25519_node_keys) assert tx_vote1.validate(b_mock) - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote1]) + assert not valid_upsert_validator_election.has_concluded(b_mock, [tx_vote1]) # 2/3 is achieved in the same block so the election can be.has_concludedd - assert ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote1, tx_vote2]) + assert valid_upsert_validator_election.has_concluded(b_mock, [tx_vote1, tx_vote2]) b_mock.store_bulk_transactions([tx_vote1]) - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) + assert not valid_upsert_validator_election.has_concluded(b_mock) assert tx_vote2.validate(b_mock) assert tx_vote3.validate(b_mock) # conclusion can be triggered my different votes in the same block - assert ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote2]) - assert ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote2, tx_vote3]) + assert valid_upsert_validator_election.has_concluded(b_mock, [tx_vote2]) + assert valid_upsert_validator_election.has_concluded(b_mock, [tx_vote2, tx_vote3]) b_mock.store_bulk_transactions([tx_vote2]) # Once the blockchain records >2/3 of the votes the election is assumed to be.has_concludedd # so any invocation of `.has_concluded` for that election should return False - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id) + assert not valid_upsert_validator_election.has_concluded(b_mock) # Vote is still valid but the election cannot be.has_concludedd as it it assmed that it has # been.has_concludedd before assert tx_vote3.validate(b_mock) - assert not ValidatorElection.has_concluded(b_mock, valid_upsert_validator_election.id, [tx_vote3]) + assert not valid_upsert_validator_election.has_concluded(b_mock, [tx_vote3]) @pytest.mark.abci @@ -285,9 +285,9 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): tx_vote1 = gen_vote(election, 1, ed25519_node_keys) tx_vote2 = gen_vote(election, 2, ed25519_node_keys) - assert not ValidatorElection.has_concluded(b, election.id, [tx_vote0]) - assert not ValidatorElection.has_concluded(b, election.id, [tx_vote0, tx_vote1]) - assert ValidatorElection.has_concluded(b, election.id, [tx_vote0, tx_vote1, tx_vote2]) + assert not election.has_concluded(b, [tx_vote0]) + assert not election.has_concluded(b, [tx_vote0, tx_vote1]) + assert election.has_concluded(b, [tx_vote0, tx_vote1, tx_vote2]) assert Election.approved_elections(b, 4, [tx_vote0]) == [] assert Election.approved_elections(b, 4, [tx_vote0, tx_vote1]) == [] @@ -297,13 +297,6 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') assert update_public_key == public_key64 - b.store_bulk_transactions([tx_vote0, tx_vote1]) - - update = Election.approved_elections(b, 4, [tx_vote2]) - assert len(update) == 1 - update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') - assert update_public_key == public_key64 - # remove validator power = 0 new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'}, diff --git a/tests/utils.py b/tests/utils.py index 852be5e6..5b2e1ea6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,12 +2,17 @@ # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) # Code is Apache-2.0 and docs are CC-BY-4.0 +import base58 +import base64 +import random + from functools import singledispatch -from bigchaindb import Vote from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection from bigchaindb.backend.schema import TABLES -from bigchaindb.elections.election import Election +from bigchaindb.common import crypto +from bigchaindb.elections.election import Election, Vote +from bigchaindb.tendermint_utils import key_to_base64 @singledispatch @@ -24,7 +29,6 @@ def flush_localmongo_db(connection, dbname): def generate_block(bigchain): from bigchaindb.common.crypto import generate_key_pair from bigchaindb.models import Transaction - import time alice = generate_key_pair() tx = Transaction.create([alice.public_key], @@ -34,7 +38,6 @@ def generate_block(bigchain): code, message = bigchain.write_transaction(tx, 'broadcast_tx_commit') assert code == 202 - time.sleep(2) def to_inputs(election, i, ed25519_node_keys): @@ -52,3 +55,56 @@ def gen_vote(election, i, ed25519_node_keys): [([election_pub_key], votes_i)], election_id=election.id)\ .sign([key_i.private_key]) + + +def generate_validators(powers): + """Generates an arbitrary number of validators with random public keys. + + The object under the `storage` key is in the format expected by DB. + + The object under the `eleciton` key is in the format expected by + the upsert validator election. + + `public_key`, `private_key` are in the format used for signing transactions. + + Args: + powers: A list of intergers representing the voting power to + assign to the corresponding validators. + """ + validators = [] + for power in powers: + kp = crypto.generate_key_pair() + validators.append({ + 'storage': { + 'public_key': { + 'value': key_to_base64(base58.b58decode(kp.public_key).hex()), + 'type': 'ed25519-base64', + }, + 'voting_power': power, + }, + 'election': { + 'node_id': f'node-{random.choice(range(100))}', + 'power': power, + 'public_key': { + 'value': base64.b16encode(base58.b58decode(kp.public_key)).decode('utf-8'), + 'type': 'ed25519-base16', + }, + }, + 'public_key': kp.public_key, + 'private_key': kp.private_key, + }) + return validators + + +def generate_election(b, cls, public_key, private_key, asset_data): + voters = cls.recipients(b) + election = cls.generate([public_key], + voters, + asset_data, + None).sign([private_key]) + + votes = [Vote.generate([election.to_inputs()[i]], + [([Election.to_public_key(election.id)], power)], + election.id) for i, (_, power) in enumerate(voters)] + + return election, votes From 2f6bbaeb4ba2409a3c0e21ba479c95b2c619ed7e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 20 Sep 2018 09:54:34 +0200 Subject: [PATCH 09/13] Link to Alberto's tips from Troubleshooting page (#2562) because they are useful! --- .../server/source/simple-deployment-template/troubleshooting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/server/source/simple-deployment-template/troubleshooting.md b/docs/server/source/simple-deployment-template/troubleshooting.md index 2d0311fd..b2d95722 100644 --- a/docs/server/source/simple-deployment-template/troubleshooting.md +++ b/docs/server/source/simple-deployment-template/troubleshooting.md @@ -26,4 +26,6 @@ addr_book_strict = false ## Other Problems +See the [Tendermint tips in the vrde/notes repository](https://github.com/vrde/notes/tree/master/tendermint). + If you're stuck, maybe [file a new issue on GitHub](https://github.com/bigchaindb/bigchaindb/issues/new). If your problem occurs often enough, we'll write about it here. From 407571ddf46405f8bdb623a6cb2be8931f6e6451 Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Thu, 20 Sep 2018 10:31:05 +0200 Subject: [PATCH 10/13] Problem: send function has a deprecation warning in the driver (#2547) Solution: replace send with the matching new functions --- acceptance/python/src/test_basic.py | 4 ++-- acceptance/python/src/test_divisible_asset.py | 7 +++---- acceptance/python/src/test_double_spend.py | 2 +- acceptance/python/src/test_multiple_owners.py | 5 ++--- acceptance/python/src/test_naughty_strings.py | 2 +- acceptance/python/src/test_stream.py | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/acceptance/python/src/test_basic.py b/acceptance/python/src/test_basic.py index 8702ef65..a6f2b571 100644 --- a/acceptance/python/src/test_basic.py +++ b/acceptance/python/src/test_basic.py @@ -61,7 +61,7 @@ def test_basic(): bike_id = fulfilled_creation_tx['id'] # Now she is ready to send it to the BigchainDB Network. - sent_transfer_tx = bdb.transactions.send(fulfilled_creation_tx) + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_creation_tx) # And just to be 100% sure, she also checks if she can retrieve # it from the BigchainDB node. @@ -107,7 +107,7 @@ def test_basic(): private_keys=alice.private_key) # She finally sends the transaction to a BigchainDB node. - sent_transfer_tx = bdb.transactions.send(fulfilled_transfer_tx) + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx) # And just to be 100% sure, she also checks if she can retrieve # it from the BigchainDB node. diff --git a/acceptance/python/src/test_divisible_asset.py b/acceptance/python/src/test_divisible_asset.py index b9a725d9..221226ee 100644 --- a/acceptance/python/src/test_divisible_asset.py +++ b/acceptance/python/src/test_divisible_asset.py @@ -74,7 +74,7 @@ def test_divisible_assets(): prepared_token_tx, private_keys=alice.private_key) - bdb.transactions.send(fulfilled_token_tx, mode='commit') + bdb.transactions.send_commit(fulfilled_token_tx) # We store the `id` of the transaction to use it later on. bike_token_id = fulfilled_token_tx['id'] @@ -116,8 +116,7 @@ def test_divisible_assets(): prepared_transfer_tx, private_keys=bob.private_key) - sent_transfer_tx = bdb.transactions.send(fulfilled_transfer_tx, - mode='commit') + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx) # First, Bob checks if the transaction was successful. assert bdb.transactions.retrieve( @@ -167,7 +166,7 @@ def test_divisible_assets(): # Remember Bob, last time you spent 3 tokens already, # so you only have 7 left. with pytest.raises(BadRequest) as error: - bdb.transactions.send(fulfilled_transfer_tx, mode='commit') + bdb.transactions.send_commit(fulfilled_transfer_tx) # Now Bob gets an error saying that the amount he wanted to spent is # higher than the amount of tokens he has left. diff --git a/acceptance/python/src/test_double_spend.py b/acceptance/python/src/test_double_spend.py index 2b9541b4..fd6da187 100644 --- a/acceptance/python/src/test_double_spend.py +++ b/acceptance/python/src/test_double_spend.py @@ -30,7 +30,7 @@ def test_double_create(): def send_and_queue(tx): try: - bdb.transactions.send(tx) + bdb.transactions.send_commit(tx) results.put('OK') except bigchaindb_driver.exceptions.TransportError as e: results.put('FAIL') diff --git a/acceptance/python/src/test_multiple_owners.py b/acceptance/python/src/test_multiple_owners.py index 89e93860..9eda2c87 100644 --- a/acceptance/python/src/test_multiple_owners.py +++ b/acceptance/python/src/test_multiple_owners.py @@ -64,7 +64,7 @@ def test_multiple_owners(): prepared_dw_tx, private_keys=[alice.private_key, bob.private_key]) - bdb.transactions.send(fulfilled_dw_tx, mode='commit') + bdb.transactions.send_commit(fulfilled_dw_tx) # We store the `id` of the transaction to use it later on. dw_id = fulfilled_dw_tx['id'] @@ -109,8 +109,7 @@ def test_multiple_owners(): prepared_transfer_tx, private_keys=[alice.private_key, bob.private_key]) - sent_transfer_tx = bdb.transactions.send(fulfilled_transfer_tx, - mode='commit') + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx) # They check if the transaction was successful. assert bdb.transactions.retrieve( diff --git a/acceptance/python/src/test_naughty_strings.py b/acceptance/python/src/test_naughty_strings.py index df4cb7dc..13cd765e 100644 --- a/acceptance/python/src/test_naughty_strings.py +++ b/acceptance/python/src/test_naughty_strings.py @@ -54,7 +54,7 @@ def send_naughty_tx(asset, metadata): # The fulfilled tx gets sent to the BDB network try: - sent_transaction = bdb.transactions.send(fulfilled_transaction) + sent_transaction = bdb.transactions.send_commit(fulfilled_transaction) except BadRequest as e: sent_transaction = e diff --git a/acceptance/python/src/test_stream.py b/acceptance/python/src/test_stream.py index e8c12c5a..5421877b 100644 --- a/acceptance/python/src/test_stream.py +++ b/acceptance/python/src/test_stream.py @@ -100,7 +100,7 @@ def test_stream(): # transactions to be in the shared queue: this is a two phase test, # first we send a bunch of transactions, then we check if they are # valid (and, in this case, they should). - bdb.transactions.send(tx, mode='async') + bdb.transactions.send_async(tx) # The `id` of every sent transaction is then stored in a list. sent.append(tx['id']) From eb139fba000c2e96dbaa12e464e7d5bac544e0e4 Mon Sep 17 00:00:00 2001 From: Muawia Khan Date: Thu, 20 Sep 2018 14:30:30 +0200 Subject: [PATCH 11/13] Remove benchmark logging (#2565). * Problem: Benchmark logging is not needed. * Solution: Clean it up. --- bigchaindb/core.py | 10 ++----- bigchaindb/log.py | 30 +------------------ docs/server/source/appendices/log-rotation.md | 1 - .../source/server-reference/configuration.md | 4 +-- setup.py | 6 +--- tests/conftest.py | 5 ---- 6 files changed, 7 insertions(+), 49 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 8be3cdaf..d7b98183 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -146,16 +146,13 @@ class App(BaseApplication): self.abort_if_abci_chain_is_not_synced() - logger.benchmark('CHECK_TX_INIT') logger.debug('check_tx: %s', raw_transaction) transaction = decode_transaction(raw_transaction) if self.bigchaindb.is_valid_transaction(transaction): logger.debug('check_tx: VALID') - logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) return ResponseCheckTx(code=CodeTypeOk) else: logger.debug('check_tx: INVALID') - logger.benchmark('CHECK_TX_END, tx_id:%s', transaction['id']) return ResponseCheckTx(code=CodeTypeError) def begin_block(self, req_begin_block): @@ -167,9 +164,9 @@ class App(BaseApplication): self.abort_if_abci_chain_is_not_synced() chain_shift = 0 if self.chain is None else self.chain['height'] - logger.benchmark('BEGIN BLOCK, height:%s, num_txs:%s', - req_begin_block.header.height + chain_shift, - req_begin_block.header.num_txs) + logger.debug('BEGIN BLOCK, height:%s, num_txs:%s', + req_begin_block.header.height + chain_shift, + req_begin_block.header.num_txs) self.block_txn_ids = [] self.block_transactions = [] @@ -253,7 +250,6 @@ class App(BaseApplication): logger.debug('Commit-ing new block with hash: apphash=%s ,' 'height=%s, txn ids=%s', data, self.new_height, self.block_txn_ids) - logger.benchmark('COMMIT_BLOCK, height:%s', self.new_height) if self.events_queue: event = Event(EventTypes.BLOCK_VALID, { diff --git a/bigchaindb/log.py b/bigchaindb/log.py index 6b6ba931..301feea3 100644 --- a/bigchaindb/log.py +++ b/bigchaindb/log.py @@ -11,8 +11,6 @@ import os DEFAULT_LOG_DIR = os.getcwd() -BENCHMARK_LOG_LEVEL = 15 - DEFAULT_LOGGING_CONFIG = { 'version': 1, @@ -29,11 +27,6 @@ DEFAULT_LOGGING_CONFIG = { 'format': ('[%(asctime)s] [%(levelname)s] (%(name)s) ' '%(message)s (%(processName)-10s - pid: %(process)d)'), 'datefmt': '%Y-%m-%d %H:%M:%S', - }, - 'benchmark': { - 'class': 'logging.Formatter', - 'format': ('%(asctime)s, %(levelname)s, %(message)s'), - 'datefmt': '%Y-%m-%d %H:%M:%S', } }, 'handlers': { @@ -59,31 +52,16 @@ DEFAULT_LOGGING_CONFIG = { 'backupCount': 5, 'formatter': 'file', 'level': logging.ERROR, - }, - 'benchmark': { - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(DEFAULT_LOG_DIR, 'bigchaindb-benchmark.log'), - 'mode': 'w', - 'maxBytes': 209715200, - 'backupCount': 5, - 'formatter': 'benchmark', - 'level': BENCHMARK_LOG_LEVEL, } }, 'loggers': {}, 'root': { 'level': logging.DEBUG, - 'handlers': ['console', 'file', 'errors', 'benchmark'], + 'handlers': ['console', 'file', 'errors'], }, } -def benchmark(self, message, *args, **kws): - # Yes, logger takes its '*args' as 'args'. - if self.isEnabledFor(BENCHMARK_LOG_LEVEL): - self._log(BENCHMARK_LOG_LEVEL, message, args, **kws) - - def _normalize_log_level(level): try: return level.upper() @@ -104,11 +82,6 @@ def setup_logging(): """ - # Add a new logging level for logging benchmark - logging.addLevelName(BENCHMARK_LOG_LEVEL, 'BENCHMARK') - logging.BENCHMARK = BENCHMARK_LOG_LEVEL - logging.Logger.benchmark = benchmark - logging_configs = DEFAULT_LOGGING_CONFIG new_logging_configs = bigchaindb.config['log'] @@ -127,7 +100,6 @@ def setup_logging(): if 'level_logfile' in new_logging_configs: level = _normalize_log_level(new_logging_configs['level_logfile']) logging_configs['handlers']['file']['level'] = level - logging_configs['handlers']['benchmark']['level'] = level if 'fmt_console' in new_logging_configs: fmt = new_logging_configs['fmt_console'] diff --git a/docs/server/source/appendices/log-rotation.md b/docs/server/source/appendices/log-rotation.md index 25c0cd6e..95d52115 100644 --- a/docs/server/source/appendices/log-rotation.md +++ b/docs/server/source/appendices/log-rotation.md @@ -25,7 +25,6 @@ please refer to [MongoDB docs](https://docs.mongodb.com/v3.6/tutorial/rotate-log Log rotation is baked into BigchainDB server using the `logging` module. BigchainDB server logs information into the following files: - `bigchaindb.log` - `bigchaindb-errors.log` - - `bigchaindb-benchmark.log` These log files are created by default in the directory from where you run `bigchaindb start`, if you are using `monit`, from [How to Set Up a BigchainDB Network](../simple-deployment-template/network-setup.md) guide, the default directory is: `$HOME/.bigchaindb-monit/logs` diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 47d06de8..66d9f23f 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -292,7 +292,7 @@ defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), but case-insensitive for the sake of convenience: ```text -"critical", "error", "warning", "info", "benchmark", "debug", "notset" +"critical", "error", "warning", "info", "debug", "notset" ``` ### log.level_logfile @@ -302,7 +302,7 @@ defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), but case-insensitive for the sake of convenience: ```text -"critical", "error", "warning", "info", "benchmark", "debug", "notset" +"critical", "error", "warning", "info", "debug", "notset" ``` ### log.datefmt_console diff --git a/setup.py b/setup.py index 4181bddf..3bb2151c 100644 --- a/setup.py +++ b/setup.py @@ -71,10 +71,6 @@ tests_require = [ 'tox', ] + docs_require -benchmarks_require = [ - 'line-profiler==1.0', -] - install_requires = [ # TODO Consider not installing the db drivers, or putting them in extras. 'pymongo~=3.6', @@ -144,7 +140,7 @@ setup( tests_require=tests_require, extras_require={ 'test': tests_require, - 'dev': dev_require + tests_require + docs_require + benchmarks_require, + 'dev': dev_require + tests_require + docs_require, 'docs': docs_require, }, package_data={'bigchaindb.common.schema': ['*.yaml']}, diff --git a/tests/conftest.py b/tests/conftest.py index 90d54dea..e001465c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,6 @@ from pymongo import MongoClient from bigchaindb import ValidatorElection from bigchaindb.common import crypto -from bigchaindb.log import setup_logging from bigchaindb.tendermint_utils import key_from_base64 from bigchaindb.backend import schema, query from bigchaindb.common.crypto import (key_pair_from_ed25519_key, @@ -108,10 +107,6 @@ def _configure_bigchaindb(request): config = config_utils.env_config(config) config_utils.set_config(config) - # NOTE: since we use a custom log level - # for benchmark logging we need to setup logging - setup_logging() - @pytest.fixture(scope='session') def _setup_database(_configure_bigchaindb): From 39be7a2fdfdd2ba0818c5598229fffbe6bbd9509 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 20 Sep 2018 17:17:33 +0200 Subject: [PATCH 12/13] Renamed "consensus" to "validation" where relevant (#2561) --- bigchaindb/README.md | 4 ++-- bigchaindb/config_utils.py | 22 +++++++++++----------- bigchaindb/core.py | 3 +-- bigchaindb/lib.py | 10 +++++----- bigchaindb/{consensus.py => validation.py} | 15 +++++++-------- tests/test_config_utils.py | 16 ++++++++-------- tests/test_core.py | 8 ++++---- 7 files changed, 38 insertions(+), 40 deletions(-) rename bigchaindb/{consensus.py => validation.py} (55%) diff --git a/bigchaindb/README.md b/bigchaindb/README.md index f3353108..61a0e8cd 100644 --- a/bigchaindb/README.md +++ b/bigchaindb/README.md @@ -18,9 +18,9 @@ The `BigchainDB` class is defined here. Most node-level operations and database `Block`, `Transaction`, and `Asset` classes are defined here. The classes mirror the block and transaction structure from the [documentation](https://docs.bigchaindb.com/projects/server/en/latest/data-models/index.html), but also include methods for validation and signing. -### [`consensus.py`](./consensus.py) +### [`validation.py`](./validation.py) -Base class for consensus methods (verification of votes, blocks, and transactions). The actual logic is mostly found in `transaction` and `block` models, defined in [`models.py`](./models.py). +Base class for validation methods (verification of votes, blocks, and transactions). The actual logic is mostly found in `transaction` and `block` models, defined in [`models.py`](./models.py). ### [`processes.py`](./processes.py) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 3836f437..3c2015f3 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -28,7 +28,7 @@ from bigchaindb.common import exceptions import bigchaindb -from bigchaindb.consensus import BaseConsensusRules +from bigchaindb.validation import BaseValidationRules # TODO: move this to a proper configuration file for logging logging.getLogger('requests').setLevel(logging.WARNING) @@ -258,38 +258,38 @@ def autoconfigure(filename=None, config=None, force=False): @lru_cache() -def load_consensus_plugin(name=None): - """Find and load the chosen consensus plugin. +def load_validation_plugin(name=None): + """Find and load the chosen validation plugin. Args: name (string): the name of the entry_point, as advertised in the setup.py of the providing package. Returns: - an uninstantiated subclass of ``bigchaindb.consensus.AbstractConsensusRules`` + an uninstantiated subclass of ``bigchaindb.validation.AbstractValidationRules`` """ if not name: - return BaseConsensusRules + return BaseValidationRules - # TODO: This will return the first plugin with group `bigchaindb.consensus` + # TODO: This will return the first plugin with group `bigchaindb.validation` # and name `name` in the active WorkingSet. # We should probably support Requirements specs in the config, e.g. - # consensus_plugin: 'my-plugin-package==0.0.1;default' + # validation_plugin: 'my-plugin-package==0.0.1;default' plugin = None - for entry_point in iter_entry_points('bigchaindb.consensus', name): + for entry_point in iter_entry_points('bigchaindb.validation', name): plugin = entry_point.load() # No matching entry_point found if not plugin: raise ResolutionError( - 'No plugin found in group `bigchaindb.consensus` with name `{}`'. + 'No plugin found in group `bigchaindb.validation` with name `{}`'. format(name)) # Is this strictness desireable? # It will probably reduce developer headaches in the wild. - if not issubclass(plugin, (BaseConsensusRules,)): + if not issubclass(plugin, (BaseValidationRules,)): raise TypeError('object of type "{}" does not implement `bigchaindb.' - 'consensus.BaseConsensusRules`'.format(type(plugin))) + 'validation.BaseValidationRules`'.format(type(plugin))) return plugin diff --git a/bigchaindb/core.py b/bigchaindb/core.py index d7b98183..34111c47 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -40,8 +40,7 @@ class App(BaseApplication): """Bridge between BigchainDB and Tendermint. The role of this class is to expose the BigchainDB - transactional logic to the Tendermint Consensus - State Machine. + transaction logic to Tendermint Core. """ def __init__(self, bigchaindb=None, events_queue=None): diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py index 2dadcb90..974a7ac5 100644 --- a/bigchaindb/lib.py +++ b/bigchaindb/lib.py @@ -27,7 +27,7 @@ from bigchaindb.common.exceptions import (SchemaValidationError, DoubleSpend) from bigchaindb.tendermint_utils import encode_transaction, merkleroot from bigchaindb import exceptions as core_exceptions -from bigchaindb.consensus import BaseConsensusRules +from bigchaindb.validation import BaseValidationRules logger = logging.getLogger(__name__) @@ -64,12 +64,12 @@ class BigchainDB(object): self.tendermint_port = bigchaindb.config['tendermint']['port'] self.endpoint = 'http://{}:{}/'.format(self.tendermint_host, self.tendermint_port) - consensusPlugin = bigchaindb.config.get('consensus_plugin') + validationPlugin = bigchaindb.config.get('validation_plugin') - if consensusPlugin: - self.consensus = config_utils.load_consensus_plugin(consensusPlugin) + if validationPlugin: + self.validation = config_utils.load_validation_plugin(validationPlugin) else: - self.consensus = BaseConsensusRules + self.validation = BaseValidationRules self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) diff --git a/bigchaindb/consensus.py b/bigchaindb/validation.py similarity index 55% rename from bigchaindb/consensus.py rename to bigchaindb/validation.py index a5a20efb..f3ec9dd6 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/validation.py @@ -3,23 +3,22 @@ # Code is Apache-2.0 and docs are CC-BY-4.0 -class BaseConsensusRules(): - """Base consensus rules for Bigchain. +class BaseValidationRules(): + """Base validation rules for BigchainDB. - A consensus plugin must expose a class inheriting from this one via an entry_point. + A validation plugin must expose a class inheriting from this one via an entry_point. All methods listed below must be implemented. - """ @staticmethod - def validate_transaction(bigchain, transaction): + def validate_transaction(bigchaindb, transaction): """See :meth:`bigchaindb.models.Transaction.validate` for documentation. """ - return transaction.validate(bigchain) + return transaction.validate(bigchaindb) @staticmethod - def validate_block(bigchain, block): + def validate_block(bigchaindb, block): """See :meth:`bigchaindb.models.Block.validate` for documentation.""" - return block.validate(bigchain) + return block.validate(bigchaindb) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 72dc4e29..af83db5b 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -31,24 +31,24 @@ def test_bigchain_instance_is_initialized_when_conf_provided(): assert bigchaindb.config['CONFIGURED'] is True -def test_load_consensus_plugin_loads_default_rules_without_name(): +def test_load_validation_plugin_loads_default_rules_without_name(): from bigchaindb import config_utils - from bigchaindb.consensus import BaseConsensusRules + from bigchaindb.validation import BaseValidationRules - assert config_utils.load_consensus_plugin() == BaseConsensusRules + assert config_utils.load_validation_plugin() == BaseValidationRules -def test_load_consensus_plugin_raises_with_unknown_name(): +def test_load_validation_plugin_raises_with_unknown_name(): from pkg_resources import ResolutionError from bigchaindb import config_utils with pytest.raises(ResolutionError): - config_utils.load_consensus_plugin('bogus') + config_utils.load_validation_plugin('bogus') -def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): +def test_load_validation_plugin_raises_with_invalid_subclass(monkeypatch): # Monkeypatch entry_point.load to return something other than a - # ConsensusRules instance + # ValidationRules instance from bigchaindb import config_utils import time monkeypatch.setattr(config_utils, @@ -58,7 +58,7 @@ def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): with pytest.raises(TypeError): # Since the function is decorated with `lru_cache`, we need to # "miss" the cache using a name that has not been used previously - config_utils.load_consensus_plugin(str(time.time())) + config_utils.load_validation_plugin(str(time.time())) def test_load_events_plugins(monkeypatch): diff --git a/tests/test_core.py b/tests/test_core.py index 72e5bc1d..85fc7ec3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -35,20 +35,20 @@ def config(request, monkeypatch): def test_bigchain_class_default_initialization(config): from bigchaindb import BigchainDB - from bigchaindb.consensus import BaseConsensusRules + from bigchaindb.validation import BaseValidationRules from bigchaindb.backend.connection import Connection bigchain = BigchainDB() assert isinstance(bigchain.connection, Connection) assert bigchain.connection.host == config['database']['host'] assert bigchain.connection.port == config['database']['port'] assert bigchain.connection.dbname == config['database']['name'] - assert bigchain.consensus == BaseConsensusRules + assert bigchain.validation == BaseValidationRules def test_bigchain_class_initialization_with_parameters(): from bigchaindb import BigchainDB from bigchaindb.backend import connect - from bigchaindb.consensus import BaseConsensusRules + from bigchaindb.validation import BaseValidationRules init_db_kwargs = { 'backend': 'localmongodb', 'host': 'this_is_the_db_host', @@ -61,7 +61,7 @@ def test_bigchain_class_initialization_with_parameters(): assert bigchain.connection.host == init_db_kwargs['host'] assert bigchain.connection.port == init_db_kwargs['port'] assert bigchain.connection.dbname == init_db_kwargs['name'] - assert bigchain.consensus == BaseConsensusRules + assert bigchain.validation == BaseValidationRules def test_get_spent_issue_1271(b, alice, bob, carol): From 24ca0b32a9f3711b1bb0d0f3aefda74644252946 Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Fri, 21 Sep 2018 10:51:57 +0200 Subject: [PATCH 13/13] Separate pending and effective validator updates. (#2556) * Separate pending and effective validator updates. - Pending validator updates do not prevent elections from concluding. - ValidatorElection overrides has_conclude to not conclude when there is a pending update for the matching height. - Effective validator updates deem past elections inconclusive. * Problem: Looking for election block is inefficient. Solution: Record placed elections, update the records upon election conclusion. * Clarify the conclusion order in Election.process_blocks. * Insert election records in bulk. Otherwise, one can significantly slow nodes down by posting a whole bunch of unique elections. * Change get_election to use find_one. * Calculate total votes without making extra query. * Fix the pending valset check. * Fix election test setup. --- bigchaindb/backend/localmongodb/query.py | 23 ++- bigchaindb/backend/localmongodb/schema.py | 3 +- bigchaindb/backend/query.py | 13 +- bigchaindb/core.py | 9 +- bigchaindb/elections/election.py | 134 ++++++++++------- bigchaindb/lib.py | 16 +- .../migrations/chain_migration_election.py | 4 +- .../upsert_validator/validator_election.py | 28 ++-- tests/backend/localmongodb/test_schema.py | 49 +----- tests/conftest.py | 9 +- tests/elections/test_election.py | 141 +++++++++++++++--- tests/upsert_validator/conftest.py | 6 +- .../test_upsert_validator_vote.py | 8 +- 13 files changed, 272 insertions(+), 171 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 6b0793ba..d193ebb9 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -282,16 +282,26 @@ def store_validator_set(conn, validators_update): @register_query(LocalMongoDBConnection) -def store_election_results(conn, election): +def store_election(conn, election_id, height, is_concluded): return conn.run( conn.collection('elections').replace_one( - {'election_id': election['election_id']}, - election, + {'election_id': election_id, + 'height': height}, + {'election_id': election_id, + 'height': height, + 'is_concluded': is_concluded}, upsert=True, ) ) +@register_query(LocalMongoDBConnection) +def store_elections(conn, elections): + return conn.run( + conn.collection('elections').insert_many(elections) + ) + + @register_query(LocalMongoDBConnection) def get_validator_set(conn, height=None): query = {} @@ -312,13 +322,12 @@ def get_validator_set(conn, height=None): def get_election(conn, election_id): query = {'election_id': election_id} - cursor = conn.run( + return conn.run( conn.collection('elections') - .find(query, projection={'_id': False}) + .find_one(query, projection={'_id': False}, + sort=[('height', DESCENDING)]) ) - return next(cursor, None) - @register_query(LocalMongoDBConnection) def get_asset_tokens_for_public_key(conn, asset_id, public_key): diff --git a/bigchaindb/backend/localmongodb/schema.py b/bigchaindb/backend/localmongodb/schema.py index 053cfc3d..157ff39d 100644 --- a/bigchaindb/backend/localmongodb/schema.py +++ b/bigchaindb/backend/localmongodb/schema.py @@ -45,7 +45,8 @@ INDEXES = { ('commit_id', dict(name='pre_commit_id', unique=True)), ], 'elections': [ - ('election_id', dict(name='election_id', unique=True)), + ([('height', DESCENDING), ('election_id', ASCENDING)], + dict(name='election_id_height', unique=True)), ], 'validators': [ ('height', dict(name='height', unique=True)), diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index d8f60320..0b5bbade 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -352,8 +352,15 @@ def store_validator_set(conn, validator_update): @singledispatch -def store_election_results(conn, election): - """Store election results""" +def store_election(conn, election_id, height, is_concluded): + """Store election record""" + + raise NotImplementedError + + +@singledispatch +def store_elections(conn, elections): + """Store election records in bulk""" raise NotImplementedError @@ -369,7 +376,7 @@ def get_validator_set(conn, height): @singledispatch def get_election(conn, election_id): - """Return a validator set change with the specified election_id + """Return the election record """ raise NotImplementedError diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 34111c47..afe646bc 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -215,10 +215,9 @@ class App(BaseApplication): else: self.block_txn_hash = block['app_hash'] - # Process all concluded elections in the current block and get any update to the validator set - update = Election.approved_elections(self.bigchaindb, - self.new_height, - self.block_transactions) + validator_update = Election.process_block(self.bigchaindb, + self.new_height, + self.block_transactions) # Store pre-commit state to recover in case there is a crash during `commit` pre_commit_state = PreCommitState(commit_id=PRE_COMMIT_ID, @@ -226,7 +225,7 @@ class App(BaseApplication): transactions=self.block_txn_ids) logger.debug('Updating PreCommitState: %s', self.new_height) self.bigchaindb.store_pre_commit_state(pre_commit_state._asdict()) - return ResponseEndBlock(validator_updates=update) + return ResponseEndBlock(validator_updates=validator_update) def commit(self): """Store the new height and along with block hash.""" diff --git a/bigchaindb/elections/election.py b/bigchaindb/elections/election.py index 40861a52..e8b3256e 100644 --- a/bigchaindb/elections/election.py +++ b/bigchaindb/elections/election.py @@ -1,7 +1,7 @@ # Copyright BigchainDB GmbH and BigchainDB contributors # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) # Code is Apache-2.0 and docs are CC-BY-4.0 -from collections import defaultdict +from collections import OrderedDict import base58 from uuid import uuid4 @@ -22,9 +22,13 @@ from bigchaindb.common.schema import (_validate_schema, class Election(Transaction): + """Represents election transactions. + + To implement a custom election, create a class deriving from this one + with OPERATION set to the election operation, ALLOWED_OPERATIONS + set to (OPERATION,), CREATE set to OPERATION. + """ - # NOTE: this transaction class extends create so the operation inheritance is achieved - # by setting an ELECTION_TYPE and renaming CREATE = ELECTION_TYPE and ALLOWED_OPERATIONS = (ELECTION_TYPE,) OPERATION = None # Custom validation schema TX_SCHEMA_CUSTOM = None @@ -34,7 +38,6 @@ class Election(Transaction): INCONCLUSIVE = 'inconclusive' # Vote ratio to approve an election ELECTION_THRESHOLD = 2 / 3 - CHANGES_VALIDATOR_SET = True @classmethod def get_validator_change(cls, bigchain): @@ -45,8 +48,10 @@ class Election(Transaction): 'validators': } """ - height = bigchain.get_latest_block()['height'] - return bigchain.get_validator_change(height) + latest_block = bigchain.get_latest_block() + if latest_block is None: + return None + return bigchain.get_validator_change(latest_block['height']) @classmethod def get_validators(cls, bigchain, height=None): @@ -186,49 +191,52 @@ class Election(Transaction): election_pk)) return self.count_votes(election_pk, txns, dict.get) - def has_concluded(self, bigchain, current_votes=[], height=None): + def has_concluded(self, bigchain, current_votes=[]): """Check if the election can be concluded or not. - * Elections can only be concluded if the current validator set - is exactly equal to the validator set encoded in the election outputs. + * Elections can only be concluded if the validator set has not changed + since the election was initiated. * Elections can be concluded only if the current votes form a supermajority. Custom elections may override this function and introduce additional checks. """ + if self.has_validator_set_changed(bigchain): + return False election_pk = self.to_public_key(self.id) votes_committed = self.get_commited_votes(bigchain, election_pk) votes_current = self.count_votes(election_pk, current_votes) - current_validators = self.get_validators(bigchain, height) - if self.is_same_topology(current_validators, self.outputs): - total_votes = sum(current_validators.values()) - if (votes_committed < (2/3) * total_votes) and \ - (votes_committed + votes_current >= (2/3)*total_votes): - return True + total_votes = sum(output.amount for output in self.outputs) + if (votes_committed < (2/3) * total_votes) and \ + (votes_committed + votes_current >= (2/3)*total_votes): + return True + return False def get_status(self, bigchain): - concluded = self.get_election(self.id, bigchain) - if concluded: + election = self.get_election(self.id, bigchain) + if election and election['is_concluded']: return self.CONCLUDED - latest_change = self.get_validator_change(bigchain) - latest_change_height = latest_change['height'] - election_height = bigchain.get_block_containing_tx(self.id)[0] + return self.INCONCLUSIVE if self.has_validator_set_changed(bigchain) else self.ONGOING - if latest_change_height >= election_height: - return self.INCONCLUSIVE - else: - return self.ONGOING + def has_validator_set_changed(self, bigchain): + latest_change = self.get_validator_change(bigchain) + if latest_change is None: + return False + + latest_change_height = latest_change['height'] + + election = self.get_election(self.id, bigchain) + + return latest_change_height > election['height'] def get_election(self, election_id, bigchain): - result = bigchain.get_election(election_id) - return result + return bigchain.get_election(election_id) - @classmethod - def store_election_results(cls, bigchain, election, height): - bigchain.store_election_results(height, election) + def store(self, bigchain, height, is_concluded): + bigchain.store_election(self.id, height, is_concluded) def show_election(self, bigchain): data = self.asset['data'] @@ -243,45 +251,61 @@ class Election(Transaction): return response @classmethod - def approved_elections(cls, bigchain, new_height, txns): - elections = defaultdict(list) + def process_block(cls, bigchain, new_height, txns): + """Looks for election and vote transactions inside the block, records + and processes elections. + + Every election is recorded in the database. + + Every vote has a chance to conclude the corresponding election. When + an election is concluded, the corresponding database record is + marked as such. + + Elections and votes are processed in the order in which they + appear in the block. Elections are concluded in the order of + appearance of their first votes in the block. + + For every election concluded in the block, calls its `on_approval` + method. The returned value of the last `on_approval`, if any, + is a validator set update to be applied in one of the following blocks. + + `on_approval` methods are implemented by elections of particular type. + The method may contain side effects but should be idempotent. To account + for other concluded elections, if it requires so, the method should + rely on the database state. + """ + # elections placed in this block + initiated_elections = [] + # elections voted for in this block and their votes + elections = OrderedDict() for tx in txns: + if isinstance(tx, Election): + initiated_elections.append({'election_id': tx.id, + 'height': new_height, + 'is_concluded': False}) if not isinstance(tx, Vote): continue election_id = tx.asset['id'] + if election_id not in elections: + elections[election_id] = [] elections[election_id].append(tx) - validator_set_updated = False - validator_set_change = [] + if initiated_elections: + bigchain.store_elections(initiated_elections) + + validator_update = None for election_id, votes in elections.items(): election = bigchain.get_transaction(election_id) if election is None: continue - if not election.has_concluded(bigchain, votes, new_height): + if not election.has_concluded(bigchain, votes): continue - if election.makes_validator_set_change(): - if validator_set_updated: - continue - validator_set_change.append(election.get_validator_set_change(bigchain, new_height)) - validator_set_updated = True + validator_update = election.on_approval(bigchain, new_height) + election.store(bigchain, new_height, is_concluded=True) - election.on_approval(bigchain, election, new_height) - election.store_election_results(bigchain, election, new_height) + return [validator_update] if validator_update else [] - return validator_set_change - - def makes_validator_set_change(self): - return self.CHANGES_VALIDATOR_SET - - def get_validator_set_change(self, bigchain, new_height): - if self.makes_validator_set_change(): - return self.change_validator_set(bigchain, new_height) - - def change_validator_set(self, bigchain, new_height): - raise NotImplementedError - - @classmethod - def on_approval(cls, bigchain, election, new_height): + def on_approval(self, bigchain, new_height): raise NotImplementedError diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py index 974a7ac5..f38b0666 100644 --- a/bigchaindb/lib.py +++ b/bigchaindb/lib.py @@ -436,8 +436,7 @@ class BigchainDB(object): return [] if result is None else result['validators'] def get_election(self, election_id): - result = backend.query.get_election(self.connection, election_id) - return result + return backend.query.get_election(self.connection, election_id) def store_pre_commit_state(self, state): return backend.query.store_pre_commit_state(self.connection, state) @@ -481,13 +480,12 @@ class BigchainDB(object): self.store_abci_chain(block['height'] + 1, new_chain_id, False) - def store_election_results(self, height, election): - """Store election results - :param height: the block height at which the election concluded - :param election: a concluded election - """ - return backend.query.store_election_results(self.connection, {'height': height, - 'election_id': election.id}) + def store_election(self, election_id, height, is_concluded): + return backend.query.store_election(self.connection, election_id, + height, is_concluded) + + def store_elections(self, elections): + return backend.query.store_elections(self.connection, elections) Block = namedtuple('Block', ('app_hash', 'height', 'transactions')) diff --git a/bigchaindb/migrations/chain_migration_election.py b/bigchaindb/migrations/chain_migration_election.py index 8e80b979..129e6684 100644 --- a/bigchaindb/migrations/chain_migration_election.py +++ b/bigchaindb/migrations/chain_migration_election.py @@ -8,7 +8,6 @@ class ChainMigrationElection(Election): CREATE = OPERATION ALLOWED_OPERATIONS = (OPERATION,) TX_SCHEMA_CUSTOM = TX_SCHEMA_CHAIN_MIGRATION_ELECTION - CHANGES_VALIDATOR_SET = False def has_concluded(self, bigchaindb, *args, **kwargs): chain = bigchaindb.get_latest_abci_chain() @@ -19,6 +18,5 @@ class ChainMigrationElection(Election): return super().has_concluded(bigchaindb, *args, **kwargs) - @classmethod - def on_approval(cls, bigchain, election, new_height): + def on_approval(self, bigchain, *args, **kwargs): bigchain.migrate_abci_chain() diff --git a/bigchaindb/upsert_validator/validator_election.py b/bigchaindb/upsert_validator/validator_election.py index 856b31a1..27e0ff56 100644 --- a/bigchaindb/upsert_validator/validator_election.py +++ b/bigchaindb/upsert_validator/validator_election.py @@ -36,18 +36,28 @@ class ValidatorElection(Election): super(ValidatorElection, cls).validate_schema(tx) validate_asset_public_key(tx['asset']['data']['public_key']) - def change_validator_set(self, bigchain, new_height): - # The new validator set comes into effect from height = new_height+1 - # (upcoming changes to Tendermint will change this to height = new_height+2) + def has_concluded(self, bigchain, *args, **kwargs): + latest_block = bigchain.get_latest_block() + if latest_block is not None: + latest_block_height = latest_block['height'] + latest_validator_change = bigchain.get_validator_change()['height'] + + # TODO change to `latest_block_height + 3` when upgrading to Tendermint 0.24.0. + if latest_validator_change == latest_block_height + 2: + # do not conclude the election if there is a change assigned already + return False + + return super().has_concluded(bigchain, *args, **kwargs) + + def on_approval(self, bigchain, new_height): validator_updates = [self.asset['data']] curr_validator_set = bigchain.get_validators(new_height) updated_validator_set = new_validator_set(curr_validator_set, validator_updates) - updated_validator_set = [v for v in updated_validator_set if v['voting_power'] > 0] - bigchain.store_validator_set(new_height+1, updated_validator_set) - return encode_validator(self.asset['data']) + updated_validator_set = [v for v in updated_validator_set + if v['voting_power'] > 0] - @classmethod - def on_approval(cls, bigchain, election, new_height): - pass + # TODO change to `new_height + 2` when upgrading to Tendermint 0.24.0. + bigchain.store_validator_set(new_height + 1, updated_validator_set) + return encode_validator(self.asset['data']) diff --git a/tests/backend/localmongodb/test_schema.py b/tests/backend/localmongodb/test_schema.py index a96e7b63..c4f6669a 100644 --- a/tests/backend/localmongodb/test_schema.py +++ b/tests/backend/localmongodb/test_schema.py @@ -3,51 +3,6 @@ # Code is Apache-2.0 and docs are CC-BY-4.0 -def test_init_creates_db_tables_and_indexes(): - import bigchaindb - from bigchaindb import backend - from bigchaindb.backend.schema import init_database - - conn = backend.connect() - dbname = bigchaindb.config['database']['name'] - - # the db is set up by the fixture so we need to remove it - conn.conn.drop_database(dbname) - - init_database() - - collection_names = conn.conn[dbname].list_collection_names() - assert set(collection_names) == { - 'transactions', 'assets', 'metadata', 'blocks', 'utxos', 'pre_commit', - 'validators', 'elections', 'abci_chains', - } - - indexes = conn.conn[dbname]['assets'].index_information().keys() - assert set(indexes) == {'_id_', 'asset_id', 'text'} - - indexes = conn.conn[dbname]['transactions'].index_information().keys() - assert set(indexes) == { - '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} - - indexes = conn.conn[dbname]['blocks'].index_information().keys() - assert set(indexes) == {'_id_', 'height'} - - indexes = conn.conn[dbname]['utxos'].index_information().keys() - assert set(indexes) == {'_id_', 'utxo'} - - indexes = conn.conn[dbname]['pre_commit'].index_information().keys() - assert set(indexes) == {'_id_', 'pre_commit_id'} - - indexes = conn.conn[dbname]['validators'].index_information().keys() - assert set(indexes) == {'_id_', 'height'} - - indexes = conn.conn[dbname]['abci_chains'].index_information().keys() - assert set(indexes) == {'_id_', 'height', 'chain_id'} - - indexes = conn.conn[dbname]['elections'].index_information().keys() - assert set(indexes) == {'_id_', 'election_id'} - - def test_init_database_is_graceful_if_db_exists(): import bigchaindb from bigchaindb import backend @@ -102,8 +57,8 @@ def test_create_tables(): ('output_index', 1)] indexes = conn.conn[dbname]['elections'].index_information() - assert set(indexes.keys()) == {'_id_', 'election_id'} - assert indexes['election_id']['unique'] + assert set(indexes.keys()) == {'_id_', 'election_id_height'} + assert indexes['election_id_height']['unique'] indexes = conn.conn[dbname]['pre_commit'].index_information() assert set(indexes.keys()) == {'_id_', 'pre_commit_id'} diff --git a/tests/conftest.py b/tests/conftest.py index e001465c..a96d843f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -712,12 +712,13 @@ def valid_upsert_validator_election_2(b_mock, node_key, new_validator): def ongoing_validator_election(b, valid_upsert_validator_election, ed25519_node_keys): validators = b.get_validators(height=1) genesis_validators = {'validators': validators, - 'height': 0, - 'election_id': None} + 'height': 0} query.store_validator_set(b.connection, genesis_validators) - b.store_bulk_transactions([valid_upsert_validator_election]) - block_1 = Block(app_hash='hash_1', height=1, transactions=[valid_upsert_validator_election.id]) + query.store_election(b.connection, valid_upsert_validator_election.id, 1, + is_concluded=False) + block_1 = Block(app_hash='hash_1', height=1, + transactions=[valid_upsert_validator_election.id]) b.store_block(block_1._asdict()) return valid_upsert_validator_election diff --git a/tests/elections/test_election.py b/tests/elections/test_election.py index 0aed1816..a8c5ce98 100644 --- a/tests/elections/test_election.py +++ b/tests/elections/test_election.py @@ -9,7 +9,7 @@ from bigchaindb.upsert_validator.validator_election import ValidatorElection @pytest.mark.bdb -def test_approved_elections_concludes_all_elections(b): +def test_process_block_concludes_all_elections(b): validators = generate_validators([1] * 4) b.store_validator_set(1, [v['storage'] for v in validators]) @@ -17,28 +17,30 @@ def test_approved_elections_concludes_all_elections(b): public_key = validators[0]['public_key'] private_key = validators[0]['private_key'] - election, votes = generate_election(b, - ValidatorElection, - public_key, private_key, - new_validator['election']) - txs = [election] - total_votes = votes election, votes = generate_election(b, ChainMigrationElection, public_key, private_key, {}) + txs = [election] + total_votes = votes + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election']) txs += [election] total_votes += votes b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 1, txs) b.store_block(Block(height=1, transactions=[tx.id for tx in txs], app_hash='')._asdict()) b.store_bulk_transactions(txs) - Election.approved_elections(b, 1, total_votes) + Election.process_block(b, 2, total_votes) validators = b.get_validators() assert len(validators) == 5 @@ -53,12 +55,11 @@ def test_approved_elections_concludes_all_elections(b): } for tx in txs: - election = b.get_election(tx.id) - assert election + assert b.get_election(tx.id)['is_concluded'] @pytest.mark.bdb -def test_approved_elections_applies_only_one_validator_update(b): +def test_process_block_approves_only_one_validator_update(b): validators = generate_validators([1] * 4) b.store_validator_set(1, [v['storage'] for v in validators]) @@ -82,24 +83,123 @@ def test_approved_elections_applies_only_one_validator_update(b): txs += [election] total_votes += votes + Election.process_block(b, 1, txs) b.store_block(Block(height=1, transactions=[tx.id for tx in txs], app_hash='')._asdict()) b.store_bulk_transactions(txs) - Election.approved_elections(b, 1, total_votes) + Election.process_block(b, 2, total_votes) validators = b.get_validators() assert len(validators) == 5 assert new_validator['storage'] in validators assert another_validator['storage'] not in validators - assert b.get_election(txs[0].id) - assert not b.get_election(txs[1].id) + assert b.get_election(txs[0].id)['is_concluded'] + assert not b.get_election(txs[1].id)['is_concluded'] @pytest.mark.bdb -def test_approved_elections_applies_only_one_migration(b): +def test_process_block_approves_after_pending_validator_update(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election']) + txs = [election] + total_votes = votes + + another_validator = generate_validators([1])[0] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + another_validator['election']) + txs += [election] + total_votes += votes + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}) + + txs += [election] + total_votes += votes + + b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 1, txs) + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.process_block(b, 2, total_votes) + + validators = b.get_validators() + assert len(validators) == 5 + assert new_validator['storage'] in validators + assert another_validator['storage'] not in validators + + assert b.get_election(txs[0].id)['is_concluded'] + assert not b.get_election(txs[1].id)['is_concluded'] + assert b.get_election(txs[2].id)['is_concluded'] + + assert b.get_latest_abci_chain() == {'height': 2, + 'chain_id': 'chain-X-migrated-at-height-1', + 'is_synced': False} + + +@pytest.mark.bdb +def test_process_block_does_not_approve_after_validator_update(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election']) + txs = [election] + total_votes = votes + + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + Election.process_block(b, 1, txs) + b.store_bulk_transactions(txs) + + second_election, second_votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}) + + Election.process_block(b, 2, total_votes + [second_election]) + + b.store_block(Block(height=2, + transactions=[v.id for v in total_votes + [second_election]], + app_hash='')._asdict()) + + b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 3, second_votes) + + assert not b.get_election(second_election.id)['is_concluded'] + assert b.get_latest_abci_chain() == {'height': 1, + 'chain_id': 'chain-X', + 'is_synced': True} + + +@pytest.mark.bdb +def test_process_block_applies_only_one_migration(b): validators = generate_validators([1] * 4) b.store_validator_set(1, [v['storage'] for v in validators]) @@ -121,12 +221,13 @@ def test_approved_elections_applies_only_one_migration(b): total_votes += votes b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 1, txs) b.store_block(Block(height=1, transactions=[tx.id for tx in txs], app_hash='')._asdict()) b.store_bulk_transactions(txs) - Election.approved_elections(b, 1, total_votes) + Election.process_block(b, 1, total_votes) chain = b.get_latest_abci_chain() assert chain assert chain == { @@ -135,9 +236,9 @@ def test_approved_elections_applies_only_one_migration(b): 'chain_id': 'chain-X-migrated-at-height-1', } - assert b.get_election(txs[0].id) - assert not b.get_election(txs[1].id) + assert b.get_election(txs[0].id)['is_concluded'] + assert not b.get_election(txs[1].id)['is_concluded'] -def test_approved_elections_gracefully_handles_empty_block(b): - Election.approved_elections(b, 1, []) +def test_process_block_gracefully_handles_empty_block(b): + Election.process_block(b, 1, []) diff --git a/tests/upsert_validator/conftest.py b/tests/upsert_validator/conftest.py index 58b5e6dd..c39b9ed3 100644 --- a/tests/upsert_validator/conftest.py +++ b/tests/upsert_validator/conftest.py @@ -28,10 +28,8 @@ def fixed_seed_election(b_mock, node_key, new_validator): @pytest.fixture def concluded_election(b, ongoing_validator_election, ed25519_node_keys): - election_result = {'height': 2, - 'election_id': ongoing_validator_election.id} - - query.store_election_results(b.connection, election_result) + query.store_election(b.connection, ongoing_validator_election.id, + 2, is_concluded=True) return ongoing_validator_election diff --git a/tests/upsert_validator/test_upsert_validator_vote.py b/tests/upsert_validator/test_upsert_validator_vote.py index 7c49bd77..eeac3e66 100644 --- a/tests/upsert_validator/test_upsert_validator_vote.py +++ b/tests/upsert_validator/test_upsert_validator_vote.py @@ -289,10 +289,10 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): assert not election.has_concluded(b, [tx_vote0, tx_vote1]) assert election.has_concluded(b, [tx_vote0, tx_vote1, tx_vote2]) - assert Election.approved_elections(b, 4, [tx_vote0]) == [] - assert Election.approved_elections(b, 4, [tx_vote0, tx_vote1]) == [] + assert Election.process_block(b, 4, [tx_vote0]) == [] + assert Election.process_block(b, 4, [tx_vote0, tx_vote1]) == [] - update = Election.approved_elections(b, 4, [tx_vote0, tx_vote1, tx_vote2]) + update = Election.process_block(b, 4, [tx_vote0, tx_vote1, tx_vote2]) assert len(update) == 1 update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') assert update_public_key == public_key64 @@ -315,7 +315,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): b.store_bulk_transactions([tx_vote0, tx_vote1]) - update = Election.approved_elections(b, 9, [tx_vote2]) + update = Election.process_block(b, 9, [tx_vote2]) assert len(update) == 1 update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') assert update_public_key == public_key64