Merge branch 'master' into tx-structure

This commit is contained in:
Scott Sadler 2017-03-14 16:22:14 +01:00
commit 1f5a8582b6
20 changed files with 120 additions and 317 deletions

View File

@ -12,7 +12,7 @@ The `Bigchain` class is defined here. Most operations outlined in the [whitepap
### [`models.py`](./models.py)
`Block`, `Transaction`, and `Asset` classes are defined here. The classes mirror the block and transaction structure from the [documentation](https://docs.bigchaindb.com/projects/server/en/latest/topic-guides/models.html), but also include methods for validation and signing.
`Block`, `Transaction`, and `Asset` classes are defined here. The classes mirror the block and transaction structure from the [documentation](https://docs.bigchaindb.com/projects/server/en/latest/data-models/index.html), but also include methods for validation and signing.
### [`consensus.py`](./consensus.py)

View File

@ -1,11 +1,4 @@
import logging
from bigchaindb.utils import verify_vote_signature
from bigchaindb.common.schema import (SchemaValidationError,
validate_vote_schema)
logger = logging.getLogger(__name__)
from bigchaindb.voting import Voting
class BaseConsensusRules():
@ -16,34 +9,15 @@ class BaseConsensusRules():
All methods listed below must be implemented.
"""
voting = Voting
@staticmethod
def validate_transaction(bigchain, transaction):
"""See :meth:`bigchaindb.models.Transaction.validate`
for documentation.
"""
for documentation."""
return transaction.validate(bigchain)
@staticmethod
def validate_block(bigchain, block):
"""See :meth:`bigchaindb.models.Block.validate` for documentation."""
return block.validate(bigchain)
@staticmethod
def verify_vote(voters, signed_vote):
"""Verify the signature of a vote.
Refer to the documentation of
:func:`bigchaindb.utils.verify_signature`.
"""
if verify_vote_signature(voters, signed_vote):
try:
validate_vote_schema(signed_vote)
return True
except SchemaValidationError as exc:
logger.warning(exc)
else:
logger.warning('Vote failed signature verification: '
'%s with voters: %s', signed_vote, voters)
return False

View File

@ -1,9 +1,6 @@
import random
import math
import collections
from time import time
from itertools import compress
from bigchaindb import exceptions as core_exceptions
from bigchaindb.common import crypto, exceptions
from bigchaindb.common.utils import gen_timestamp, serialize
@ -72,6 +69,9 @@ class Bigchain(object):
if not self.me or not self.me_private:
raise exceptions.KeypairNotFoundException()
federation = property(lambda self: set(self.nodes_except_me + [self.me]))
""" Set of federation member public keys """
def write_transaction(self, signed_transaction):
"""Write the transaction to bigchain.
@ -110,19 +110,10 @@ class Bigchain(object):
dict: database response or None if no reassignment is possible
"""
if self.nodes_except_me:
try:
federation_nodes = self.nodes_except_me + [self.me]
index_current_assignee = federation_nodes.index(transaction['assignee'])
new_assignee = random.choice(federation_nodes[:index_current_assignee] +
federation_nodes[index_current_assignee + 1:])
except ValueError:
# current assignee not in federation
new_assignee = random.choice(self.nodes_except_me)
else:
# There is no other node to assign to
new_assignee = self.me
other_nodes = tuple(
self.federation.difference([transaction['assignee']])
)
new_assignee = random.choice(other_nodes) if other_nodes else self.me
return backend.query.update_transaction(
self.connection, transaction['id'],
@ -194,8 +185,7 @@ class Bigchain(object):
if include_status:
if block:
status = self.block_election_status(block_id,
block['block']['voters'])
status = self.block_election_status(block)
return block, status
else:
return block
@ -296,12 +286,8 @@ class Bigchain(object):
blocks = backend.query.get_blocks_status_from_transaction(self.connection, txid)
if blocks:
# Determine the election status of each block
validity = {
block['id']: self.block_election_status(
block['id'],
block['block']['voters']
) for block in blocks
}
validity = {block['id']: self.block_election_status(block)
for block in blocks}
# NOTE: If there are multiple valid blocks with this transaction,
# something has gone wrong
@ -464,7 +450,7 @@ class Bigchain(object):
raise exceptions.OperationError('Empty block creation is not '
'allowed')
voters = self.nodes_except_me + [self.me]
voters = list(self.federation)
block = Block(validated_transactions, self.me, gen_timestamp(), voters)
block = block.sign(self.me_private)
@ -483,36 +469,20 @@ class Bigchain(object):
"""
return self.consensus.validate_block(self, block)
def has_previous_vote(self, block_id, voters):
def has_previous_vote(self, block_id):
"""Check for previous votes from this node
Args:
block_id (str): the id of the block to check
voters (list(str)): the voters of the block to check
Returns:
bool: :const:`True` if this block already has a
valid vote from this node, :const:`False` otherwise.
Raises:
ImproperVoteError: If there is already a vote,
but the vote is invalid.
"""
votes = list(backend.query.get_votes_by_block_id_and_voter(self.connection, block_id, self.me))
if len(votes) > 1:
raise exceptions.MultipleVotesError('Block {block_id} has {n_votes} votes from public key {me}'
.format(block_id=block_id, n_votes=str(len(votes)), me=self.me))
has_previous_vote = False
if votes:
if utils.verify_vote_signature(voters, votes[0]):
has_previous_vote = True
else:
raise exceptions.ImproperVoteError('Block {block_id} already has an incorrectly signed vote '
'from public key {me}'.format(block_id=block_id, me=self.me))
return has_previous_vote
el, _ = self.consensus.voting.partition_eligible_votes(votes, [self.me])
return bool(el)
def write_block(self, block):
"""Write a block to bigchain.
@ -612,69 +582,15 @@ class Bigchain(object):
# XXX: should this return instaces of Block?
return backend.query.get_unvoted_blocks(self.connection, self.me)
def block_election_status(self, block_id, voters):
"""Tally the votes on a block, and return the status: valid, invalid, or undecided."""
def block_election(self, block):
if type(block) != dict:
block = block.to_dict()
votes = list(backend.query.get_votes_by_block_id(self.connection,
block['id']))
return self.consensus.voting.block_election(block, votes,
self.federation)
votes = list(backend.query.get_votes_by_block_id(self.connection, block_id))
n_voters = len(voters)
voter_counts = collections.Counter([vote['node_pubkey'] for vote in votes])
for node in voter_counts:
if voter_counts[node] > 1:
raise exceptions.MultipleVotesError(
'Block {block_id} has multiple votes ({n_votes}) from voting node {node_id}'
.format(block_id=block_id, n_votes=str(voter_counts[node]), node_id=node))
if len(votes) > n_voters:
raise exceptions.MultipleVotesError('Block {block_id} has {n_votes} votes cast, but only {n_voters} voters'
.format(block_id=block_id, n_votes=str(len(votes)),
n_voters=str(n_voters)))
# vote_cast is the list of votes e.g. [True, True, False]
vote_cast = [vote['vote']['is_block_valid'] for vote in votes]
# prev_block are the ids of the nominal prev blocks e.g.
# ['block1_id', 'block1_id', 'block2_id']
prev_block = [vote['vote']['previous_block'] for vote in votes]
# vote_validity checks whether a vote is valid
# or invalid, e.g. [False, True, True]
vote_validity = [self.consensus.verify_vote(voters, vote) for vote in votes]
# element-wise product of stated vote and validity of vote
# vote_cast = [True, True, False] and
# vote_validity = [False, True, True] gives
# [True, False]
# Only the correctly signed votes are tallied.
vote_list = list(compress(vote_cast, vote_validity))
# Total the votes. Here, valid and invalid refer
# to the vote cast, not whether the vote itself
# is valid or invalid.
n_valid_votes = sum(vote_list)
n_invalid_votes = len(vote_cast) - n_valid_votes
# The use of ceiling and floor is to account for the case of an
# even number of voters where half the voters have voted 'invalid'
# and half 'valid'. In this case, the block should be marked invalid
# to avoid a tie. In the case of an odd number of voters this is not
# relevant, since one side must be a majority.
if n_invalid_votes >= math.ceil(n_voters / 2):
return Bigchain.BLOCK_INVALID
elif n_valid_votes > math.floor(n_voters / 2):
# The block could be valid, but we still need to check if votes
# agree on the previous block.
#
# First, only consider blocks with legitimate votes
prev_block_list = list(compress(prev_block, vote_validity))
# Next, only consider the blocks with 'yes' votes
prev_block_valid_list = list(compress(prev_block_list, vote_list))
counts = collections.Counter(prev_block_valid_list)
# Make sure the majority vote agrees on previous node.
# The majority vote must be the most common, by definition.
# If it's not, there is no majority agreement on the previous
# block.
if counts.most_common()[0][1] > math.floor(n_voters / 2):
return Bigchain.BLOCK_VALID
else:
return Bigchain.BLOCK_INVALID
else:
return Bigchain.BLOCK_UNDECIDED
def block_election_status(self, block):
"""Tally the votes on a block, and return the status:
valid, invalid, or undecided."""
return self.block_election(block)['status']

View File

@ -180,8 +180,7 @@ class Block(object):
ValidationError: If there is a problem with the block
"""
# Check if the block was created by a federation node
possible_voters = (bigchain.nodes_except_me + [bigchain.me])
if self.node_pubkey not in possible_voters:
if self.node_pubkey not in bigchain.federation:
raise SybilError('Only federation nodes can create blocks')
# Check that the signature is valid

View File

@ -16,6 +16,7 @@ from bigchaindb import Bigchain
logger = logging.getLogger(__name__)
logger_results = logging.getLogger('pipeline.election.results')
class Election:
@ -32,14 +33,29 @@ class Election:
next_vote: The next vote.
"""
next_block = self.bigchain.get_block(
next_vote['vote']['voting_for_block'])
try:
block_id = next_vote['vote']['voting_for_block']
node = next_vote['node_pubkey']
except KeyError:
return
block_status = self.bigchain.block_election_status(next_block['id'],
next_block['block']['voters'])
if block_status == self.bigchain.BLOCK_INVALID:
next_block = self.bigchain.get_block(block_id)
result = self.bigchain.block_election(next_block)
if result['status'] == self.bigchain.BLOCK_INVALID:
return Block.from_dict(next_block)
# Log the result
if result['status'] != self.bigchain.BLOCK_UNDECIDED:
msg = 'node:%s block:%s status:%s' % \
(node, block_id, result['status'])
# Extra data can be accessed via the log formatter.
# See logging.dictConfig.
logger_results.debug(msg, extra={
'current_vote': next_vote,
'election_result': result,
})
def requeue_transactions(self, invalid_block):
"""
Liquidates transactions from invalid blocks so they can be processed again

View File

@ -48,8 +48,7 @@ class Vote:
[([self.bigchain.me], 1)])
def validate_block(self, block):
if not self.bigchain.has_previous_vote(block['id'],
block['block']['voters']):
if not self.bigchain.has_previous_vote(block['id']):
try:
block = Block.from_dict(block)
except (exceptions.InvalidHash):

View File

@ -3,9 +3,6 @@ import threading
import queue
import multiprocessing as mp
from bigchaindb.common import crypto
from bigchaindb.common.utils import serialize
class ProcessGroup(object):
@ -116,30 +113,6 @@ def condition_details_has_owner(condition_details, owner):
return False
def verify_vote_signature(voters, signed_vote):
"""Verify the signature of a vote
A valid vote should have been signed by a voter's private key.
Args:
voters (list): voters of the block that is under election
signed_vote (dict): a vote with the `signature` included.
Returns:
bool: True if the signature is correct, False otherwise.
"""
signature = signed_vote['signature']
pk_base58 = signed_vote['node_pubkey']
# immediately return False if the voter is not in the block voter list
if pk_base58 not in voters:
return False
public_key = crypto.PublicKey(pk_base58)
return public_key.verify(serialize(signed_vote['vote']).encode(), signature)
def is_genesis_block(block):
"""Check if the block is the genesis block.

View File

@ -12,14 +12,14 @@ UNDECIDED = 'undecided'
class Voting:
"""
Everything to do with creating and checking votes.
Everything to do with verifying and counting votes for block election.
All functions in this class should be referentially transparent, that is,
they always give the same output for a given input. This makes it easier
to test. This also means no logging!
Assumptions regarding data:
* Vote is a dictionary, but it is not assumed that any properties are.
* Vote is a dictionary, but no assumptions are made on it's properties.
* Everything else is assumed to be structurally correct, otherwise errors
may be thrown.
"""
@ -30,10 +30,11 @@ class Voting:
Calculate the election status of a block.
"""
eligible_voters = set(block['block']['voters']) & set(keyring)
n_voters = len(eligible_voters)
eligible_votes, ineligible_votes = \
cls.partition_eligible_votes(votes, eligible_voters)
n_voters = len(eligible_voters)
results = cls.count_votes(eligible_votes)
results['block_id'] = block['id']
results['status'] = cls.decide_votes(n_voters, **results['counts'])
results['ineligible'] = ineligible_votes
return results

View File

@ -13,7 +13,6 @@ BigchainDB Server Documentation
server-reference/index
drivers-clients/index
clusters-feds/index
topic-guides/index
data-models/index
schema/transaction
schema/vote

View File

@ -1,12 +0,0 @@
Topic Guides
============
.. note::
Most of the Topic Guides have been moved over to `the root BigchainDB project docs <https://docs.bigchaindb.com/en/latest/index.html>`_.
.. toctree::
:maxdepth: 1
models

View File

@ -1,6 +0,0 @@
# The Transaction, Block and Vote Models
This page about transaction concepts and data models was getting too big, so it was split into smaller pages. It will be deleted eventually, so update your links. Here's where you can find the new pages:
* [Transaction Concepts](https://docs.bigchaindb.com/en/latest/transaction-concepts.html)
* [Data Models (all of them)](../data-models/index.html)

View File

@ -1,7 +1,7 @@
# Toolbox container for debugging
# Run as:
# docker run -it --rm --entrypoint sh krish7919/toolbox
# kubectl run -it toolbox --image krish7919/toolbox --restart=Never --rm
# docker run -it --rm --entrypoint sh bigchaindb/toolbox
# kubectl run -it toolbox --image bigchaindb/toolbox --restart=Never --rm
FROM alpine:3.5
MAINTAINER github.com/krish7919

View File

@ -82,12 +82,12 @@ class TestBigchainApi(object):
block = b.create_block([tx])
b.write_block(block)
assert b.has_previous_vote(block.id, block.voters) is False
assert b.has_previous_vote(block.id) is False
vote = b.vote(block.id, b.get_last_voted_block().id, True)
b.write_vote(vote)
assert b.has_previous_vote(block.id, block.voters) is True
assert b.has_previous_vote(block.id) is True
@pytest.mark.genesis
def test_get_spent_with_double_inclusion_detected(self, b, monkeypatch):
@ -463,58 +463,6 @@ class TestBigchainApi(object):
assert retrieved_block_1 == retrieved_block_2
@pytest.mark.genesis
def test_more_votes_than_voters(self, b):
from bigchaindb.common.exceptions import MultipleVotesError
block_1 = dummy_block()
b.write_block(block_1)
# insert duplicate votes
vote_1 = b.vote(block_1.id, b.get_last_voted_block().id, True)
vote_2 = b.vote(block_1.id, b.get_last_voted_block().id, True)
vote_2['node_pubkey'] = 'aaaaaaa'
b.write_vote(vote_1)
b.write_vote(vote_2)
with pytest.raises(MultipleVotesError) as excinfo:
b.block_election_status(block_1.id, block_1.voters)
assert excinfo.value.args[0] == 'Block {block_id} has {n_votes} votes cast, but only {n_voters} voters'\
.format(block_id=block_1.id, n_votes=str(2), n_voters=str(1))
def test_multiple_votes_single_node(self, b, genesis_block):
from bigchaindb.common.exceptions import MultipleVotesError
block_1 = dummy_block()
b.write_block(block_1)
# insert duplicate votes
for i in range(2):
b.write_vote(b.vote(block_1.id, genesis_block.id, True))
with pytest.raises(MultipleVotesError) as excinfo:
b.block_election_status(block_1.id, block_1.voters)
assert excinfo.value.args[0] == 'Block {block_id} has multiple votes ({n_votes}) from voting node {node_id}'\
.format(block_id=block_1.id, n_votes=str(2), node_id=b.me)
with pytest.raises(MultipleVotesError) as excinfo:
b.has_previous_vote(block_1.id, block_1.voters)
assert excinfo.value.args[0] == 'Block {block_id} has {n_votes} votes from public key {me}'\
.format(block_id=block_1.id, n_votes=str(2), me=b.me)
@pytest.mark.genesis
def test_improper_vote_error(selfs, b):
from bigchaindb.common.exceptions import ImproperVoteError
block_1 = dummy_block()
b.write_block(block_1)
vote_1 = b.vote(block_1.id, b.get_last_voted_block().id, True)
# mangle the signature
vote_1['signature'] = 'a' * 87
b.write_vote(vote_1)
with pytest.raises(ImproperVoteError) as excinfo:
b.has_previous_vote(block_1.id, block_1.id)
assert excinfo.value.args[0] == 'Block {block_id} already has an incorrectly signed ' \
'vote from public key {me}'.format(block_id=block_1.id, me=b.me)
@pytest.mark.usefixtures('inputs')
def test_assign_transaction_one_node(self, b, user_pk, user_sk):
from bigchaindb.backend import query

View File

@ -83,12 +83,6 @@ def test_check_for_quorum_invalid_prev_node(b, user_pk):
def test_check_for_quorum_valid(b, user_pk):
from bigchaindb.models import Transaction
e = election.Election()
# create blocks with transactions
tx1 = Transaction.create([b.me], [([user_pk], 1)])
test_block = b.create_block([tx1])
# simulate a federation with four voters
key_pairs = [crypto.generate_key_pair() for _ in range(4)]
test_federation = [
@ -96,8 +90,13 @@ def test_check_for_quorum_valid(b, user_pk):
for key_pair in key_pairs
]
b.nodes_except_me = [key_pair[1] for key_pair in key_pairs]
# create blocks with transactions
tx1 = Transaction.create([b.me], [([user_pk], 1)])
test_block = b.create_block([tx1])
# add voters to block and write
test_block.voters = [key_pair[1] for key_pair in key_pairs]
test_block = test_block.sign(b.me_private)
b.write_block(test_block)
@ -108,10 +107,20 @@ def test_check_for_quorum_valid(b, user_pk):
for vote in votes:
b.write_vote(vote)
e = election.Election()
e.bigchain = b
# since this block is valid, should go nowhere
assert e.check_for_quorum(votes[-1]) is None
@patch('bigchaindb.core.Bigchain.get_block')
def test_invalid_vote(get_block, b):
e = election.Election()
assert e.check_for_quorum({}) is None
get_block.assert_not_called()
@pytest.mark.bdb
def test_check_requeue_transaction(b, user_pk):
from bigchaindb.models import Transaction

View File

@ -36,7 +36,11 @@ def test_reassign_transactions(b, user_pk):
stm = stale.StaleTransactionMonitor(timeout=0.001,
backlog_reassign_delay=0.001)
stm.reassign_transactions(tx.to_dict())
# This worked previously because transaction['assignee'] was only used if
# bigchain.nodes_except_me was not empty.
tx_dict = tx.to_dict()
tx_dict['assignee'] = b.me
stm.reassign_transactions(tx_dict)
# test with federation
tx = Transaction.create([b.me], [([user_pk], 1)])
@ -58,7 +62,7 @@ def test_reassign_transactions(b, user_pk):
tx = tx.sign([b.me_private])
stm.bigchain.nodes_except_me = ['lol']
b.write_transaction(tx)
stm.bigchain.nodes_except_me = None
stm.bigchain.nodes_except_me = []
tx = list(query.get_stale_transactions(b.connection, 0))[0]
stm.reassign_transactions(tx)

View File

@ -1,40 +0,0 @@
def test_verify_vote_passes(b, structurally_valid_vote):
from bigchaindb.consensus import BaseConsensusRules
from bigchaindb.common import crypto
from bigchaindb.common.utils import serialize
vote_body = structurally_valid_vote['vote']
vote_data = serialize(vote_body)
signature = crypto.PrivateKey(b.me_private).sign(vote_data.encode())
vote_signed = {
'node_pubkey': b.me,
'signature': signature.decode(),
'vote': vote_body
}
assert BaseConsensusRules.verify_vote([b.me], vote_signed)
def test_verify_vote_fails_signature(b, structurally_valid_vote):
from bigchaindb.consensus import BaseConsensusRules
vote_body = structurally_valid_vote['vote']
vote_signed = {
'node_pubkey': b.me,
'signature': 'a' * 86,
'vote': vote_body
}
assert not BaseConsensusRules.verify_vote([b.me], vote_signed)
def test_verify_vote_fails_schema(b):
from bigchaindb.consensus import BaseConsensusRules
from bigchaindb.common import crypto
from bigchaindb.common.utils import serialize
vote_body = {}
vote_data = serialize(vote_body)
signature = crypto.PrivateKey(b.me_private).sign(vote_data.encode())
vote_signed = {
'node_pubkey': b.me,
'signature': signature.decode(),
'vote': vote_body
}
assert not BaseConsensusRules.verify_vote([b.me], vote_signed)

View File

@ -80,13 +80,3 @@ def test_get_blocks_status_containing_tx(monkeypatch):
bigchain = Bigchain(public_key='pubkey', private_key='privkey')
with pytest.raises(Exception):
bigchain.get_blocks_status_containing_tx('txid')
def test_has_previous_vote(monkeypatch):
from bigchaindb.core import Bigchain
monkeypatch.setattr(
'bigchaindb.utils.verify_vote_signature', lambda voters, vote: False)
bigchain = Bigchain(public_key='pubkey', private_key='privkey')
block = {'votes': ({'node_pubkey': 'pubkey'},)}
with pytest.raises(Exception):
bigchain.has_previous_vote(block)

View File

@ -98,13 +98,12 @@ class TestBlockModel(object):
transactions = [Transaction.create([b.me], [([b.me], 1)])]
timestamp = gen_timestamp()
voters = ['Qaaa', 'Qbbb']
block = {
'timestamp': timestamp,
'transactions': [tx.to_dict() for tx in transactions],
'node_pubkey': b.me,
'voters': voters,
'voters': list(b.federation),
}
block_body = {

View File

@ -201,3 +201,37 @@ def test_verify_vote_schema(b):
assert not Voting.verify_vote_schema(vote)
vote = b.vote('b', 'a' * 64, True)
assert not Voting.verify_vote_schema(vote)
################################################################################
# block_election tests
def test_block_election(b):
class TestVoting(Voting):
@classmethod
def verify_vote_signature(cls, vote):
return True
@classmethod
def verify_vote_schema(cls, vote):
return True
keyring = 'abc'
block = {'id': 'xyz', 'block': {'voters': 'ab'}}
votes = [{
'node_pubkey': c,
'vote': {'is_block_valid': True, 'previous_block': 'a'}
} for c in 'abc']
assert TestVoting.block_election(block, votes, keyring) == {
'status': VALID,
'block_id': 'xyz',
'counts': {'n_valid': 2, 'n_invalid': 0},
'ineligible': [votes[-1]],
'cheat': [],
'malformed': [],
'previous_block': 'a',
'other_previous_block': {},
}

View File

@ -30,7 +30,7 @@ def test_get_block_status_endpoint_undecided(b, client):
block = b.create_block([tx])
b.write_block(block)
status = b.block_election_status(block.id, block.voters)
status = b.block_election_status(block)
res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id)
assert status == res.json['status']
@ -51,7 +51,7 @@ def test_get_block_status_endpoint_valid(b, client):
vote = b.vote(block.id, b.get_last_voted_block().id, True)
b.write_vote(vote)
status = b.block_election_status(block.id, block.voters)
status = b.block_election_status(block)
res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id)
assert status == res.json['status']
@ -72,7 +72,7 @@ def test_get_block_status_endpoint_invalid(b, client):
vote = b.vote(block.id, b.get_last_voted_block().id, False)
b.write_vote(vote)
status = b.block_election_status(block.id, block.voters)
status = b.block_election_status(block)
res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id)
assert status == res.json['status']