From 0d8a53cb68400ba5f68c747b4c37b2b1eacf64ca Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 13:54:09 +0100 Subject: [PATCH 01/25] vote schema --- bigchaindb/common/schema/__init__.py | 29 +++++++++---- bigchaindb/common/schema/vote.yaml | 65 ++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 bigchaindb/common/schema/vote.yaml diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 9fdb3136..a49495f9 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -7,18 +7,31 @@ import yaml from bigchaindb.common.exceptions import SchemaValidationError -TX_SCHEMA_PATH = os.path.join(os.path.dirname(__file__), 'transaction.yaml') -with open(TX_SCHEMA_PATH) as handle: - TX_SCHEMA_YAML = handle.read() -TX_SCHEMA = yaml.safe_load(TX_SCHEMA_YAML) +def _load_schema(name): + """ Load a schema from disk """ + path = os.path.join(os.path.dirname(__file__), name + '.yaml') + with open(path) as handle: + return path, yaml.safe_load(handle) -def validate_transaction_schema(tx_body): - """ Validate a transaction dict against a schema """ +def _validate_schema(schema, body): + """ Validate data against a schema """ try: - jsonschema.validate(tx_body, TX_SCHEMA) + jsonschema.validate(body, schema) except jsonschema.ValidationError as exc: raise SchemaValidationError(str(exc)) from exc -__all__ = ['TX_SCHEMA', 'TX_SCHEMA_YAML', 'validate_transaction_schema'] +TX_SCHEMA_PATH, TX_SCHEMA = _load_schema('transaction') +VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote') + + +def validate_transaction_schema(tx_body): + """ Validate a transaction dict """ + _validate_schema(TX_SCHEMA, tx_body) + + +def validate_vote_schema(tx_body): + """ Validate a vote dict """ + _validate_schema(VOTE_SCHEMA, tx_body) + diff --git a/bigchaindb/common/schema/vote.yaml b/bigchaindb/common/schema/vote.yaml new file mode 100644 index 00000000..5bd02cda --- /dev/null +++ b/bigchaindb/common/schema/vote.yaml @@ -0,0 +1,65 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +id: "http://www.bigchaindb.com/schema/vote.json" +type: object +additionalProperties: false +title: Vote Schema +description: | + A Vote is an endorsement of a Block (identified by a hash) by + a node (identified by a public key). +required: +- node_pubkey +- signature +- vote +properties: + node_pubkey: + type: "string" + pattern: "[1-9a-zA-Z^OIl]{43,44}" + description: | + Ed25519 public key identifying the voting node. + signature: + type: "string" + pattern: "[1-9a-zA-Z^OIl]{86,88}" + description: + Ed25519 signature of the ``vote`` object. + vote: + type: "object" + additionalProperties: false + description: | + Vote details to be signed. + required: + - invalid_reason + - is_block_valid + - previous_block + - voting_for_block + - timestamp + properties: + invalid_reason: + anyOf: + - type: "string" + description: | + Reason the block is voted invalid, or ``null``. + - type: "null" + is_block_valid: + type: "boolean" + description: | + This field is ``true`` if the block was deemed valid by the node. + previous_block: + "$ref": "#/definitions/sha3_hexdigest" + description: | + Sha3 identifier of the block that preceeds the block being voted on. + The notion of a "previous" block is subject to vote. + voting_for_block: + "$ref": "#/definitions/sha3_hexdigest" + description: | + Sha3 identifier of the block being voted on. + timestamp: + type: "string" + pattern: "[0-9]{10}" + description: | + Unix timestamp that the vote was created by the node, according + to the system time of the node. +definitions: + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string From 31be5b3c8aabe8d447c1072d927f2534ef02de71 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 14:39:58 +0100 Subject: [PATCH 02/25] generalize generate_schema_documentation.py --- docs/server/generate_schema_documentation.py | 116 ++++++++++++------- 1 file changed, 73 insertions(+), 43 deletions(-) diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 1d1b751b..e5562444 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -5,7 +5,7 @@ import os.path import yaml -from bigchaindb.common.schema import TX_SCHEMA_YAML +from bigchaindb.common.schema import TX_SCHEMA_PATH TPL_PROP = """\ @@ -18,7 +18,28 @@ TPL_PROP = """\ """ -TPL_DOC = """\ +TPL_STYLES = """ +.. raw:: html + + +""" + + +TPL_TRANSACTION = """\ .. This file was auto generated by %(file)s ================== @@ -35,23 +56,6 @@ Transaction Schema * Metadata_ -.. raw:: html - - Transaction ----------- @@ -77,10 +81,20 @@ Metadata -------- %(metadata)s -""" +""" + TPL_STYLES -def ordered_load_yaml(stream): +TPL_VOTE = """\ +.. This file was auto generated by %(file)s + +=========== +Vote Schema +=========== + +""" + TPL_STYLES + + +def ordered_load_yaml(path): """ Custom YAML loader to preserve key order """ class OrderedLoader(yaml.SafeLoader): pass @@ -91,13 +105,43 @@ def ordered_load_yaml(stream): OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) - return yaml.load(stream, OrderedLoader) + with open(path) as handle: + return yaml.load(handle, OrderedLoader) -TX_SCHEMA = ordered_load_yaml(TX_SCHEMA_YAML) +def load_schema(path): + global DEFS + schema = ordered_load_yaml(path) + DEFS = schema['definitions'] + return schema -DEFINITION_BASE_PATH = '#/definitions/' +def generate_transaction_docs(): + schema = load_schema(TX_SCHEMA_PATH) + defs = schema['definitions'] + + doc = TPL_TRANSACTION % { + 'transaction': render_section('Transaction', schema), + 'condition': render_section('Condition', defs['condition']), + 'fulfillment': render_section('Fulfillment', defs['fulfillment']), + 'asset': render_section('Asset', defs['asset']), + 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), + 'container': 'transaction-schema', + 'file': os.path.basename(__file__), + } + + write_schema_doc('transaction', doc) + + +def write_schema_doc(name, doc): + # Check base path exists + base_path = os.path.join(os.path.dirname(__file__), 'source/schema') + if not os.path.exists(base_path): + os.makedirs(base_path) + # Write doc + path = os.path.join(base_path, '%s.rst' % name) + with open(path, 'w') as handle: + handle.write(doc) def render_section(section_name, obj): @@ -141,32 +185,18 @@ def property_type(prop): raise ValueError("Could not resolve property type") +DEFINITION_BASE_PATH = '#/definitions/' + + def resolve_ref(ref): """ Resolve definition reference """ assert ref.startswith(DEFINITION_BASE_PATH) - return TX_SCHEMA['definitions'][ref[len(DEFINITION_BASE_PATH):]] + return DEFS[ref[len(DEFINITION_BASE_PATH):]] def main(): """ Main function """ - defs = TX_SCHEMA['definitions'] - doc = TPL_DOC % { - 'transaction': render_section('Transaction', TX_SCHEMA), - 'condition': render_section('Condition', defs['condition']), - 'fulfillment': render_section('Fulfillment', defs['fulfillment']), - 'asset': render_section('Asset', defs['asset']), - 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), - 'file': os.path.basename(__file__), - } - - base_path = os.path.join(os.path.dirname(__file__), 'source/schema') - path = os.path.join(base_path, 'transaction.rst') - - if not os.path.exists(base_path): - os.makedirs(base_path) - - with open(path, 'w') as handle: - handle.write(doc) + generate_transaction_docs() def setup(*_): From b227739d89d75df1e68a42a8e88b4ad54cfcf427 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 15:07:51 +0100 Subject: [PATCH 03/25] render schema/vote.rst --- bigchaindb/common/schema/vote.yaml | 24 ++-- docs/server/generate_schema_documentation.py | 60 +++++++--- docs/server/source/index.rst | 1 + docs/server/source/schema/vote.rst | 118 +++++++++++++++++++ 4 files changed, 175 insertions(+), 28 deletions(-) create mode 100644 docs/server/source/schema/vote.rst diff --git a/bigchaindb/common/schema/vote.yaml b/bigchaindb/common/schema/vote.yaml index 5bd02cda..8a09a10a 100644 --- a/bigchaindb/common/schema/vote.yaml +++ b/bigchaindb/common/schema/vote.yaml @@ -7,6 +7,10 @@ title: Vote Schema description: | A Vote is an endorsement of a Block (identified by a hash) by a node (identified by a public key). + + The outer Vote object contains the details of the vote being made + as well as the signature and identifying information of the node + passing the vote. required: - node_pubkey - signature @@ -34,16 +38,6 @@ properties: - voting_for_block - timestamp properties: - invalid_reason: - anyOf: - - type: "string" - description: | - Reason the block is voted invalid, or ``null``. - - type: "null" - is_block_valid: - type: "boolean" - description: | - This field is ``true`` if the block was deemed valid by the node. previous_block: "$ref": "#/definitions/sha3_hexdigest" description: | @@ -53,6 +47,16 @@ properties: "$ref": "#/definitions/sha3_hexdigest" description: | Sha3 identifier of the block being voted on. + is_block_valid: + type: "boolean" + description: | + This field is ``true`` if the block was deemed valid by the node. + invalid_reason: + anyOf: + - type: "string" + description: | + Reason the block is voted invalid, or ``null``. + - type: "null" timestamp: type: "string" pattern: "[0-9]{10}" diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index e5562444..704157f3 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -5,7 +5,7 @@ import os.path import yaml -from bigchaindb.common.schema import TX_SCHEMA_PATH +from bigchaindb.common.schema import TX_SCHEMA_PATH, VOTE_SCHEMA_PATH TPL_PROP = """\ @@ -84,6 +84,23 @@ Metadata """ + TPL_STYLES +def generate_transaction_docs(): + schema = load_schema(TX_SCHEMA_PATH) + defs = schema['definitions'] + + doc = TPL_TRANSACTION % { + 'transaction': render_section('Transaction', schema), + 'condition': render_section('Condition', defs['condition']), + 'fulfillment': render_section('Fulfillment', defs['fulfillment']), + 'asset': render_section('Asset', defs['asset']), + 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), + 'container': 'transaction-schema', + 'file': os.path.basename(__file__), + } + + write_schema_doc('transaction', doc) + + TPL_VOTE = """\ .. This file was auto generated by %(file)s @@ -91,9 +108,32 @@ TPL_VOTE = """\ Vote Schema =========== +Vote +---- + +%(vote)s + +Vote Details +------------ + +%(vote_details)s + """ + TPL_STYLES +def generate_vote_docs(): + schema = load_schema(VOTE_SCHEMA_PATH) + + doc = TPL_VOTE % { + 'vote': render_section('Vote', schema), + 'vote_details': render_section('Vote', schema['properties']['vote']), + 'container': 'vote-schema', + 'file': os.path.basename(__file__), + } + + write_schema_doc('vote', doc) + + def ordered_load_yaml(path): """ Custom YAML loader to preserve key order """ class OrderedLoader(yaml.SafeLoader): @@ -116,23 +156,6 @@ def load_schema(path): return schema -def generate_transaction_docs(): - schema = load_schema(TX_SCHEMA_PATH) - defs = schema['definitions'] - - doc = TPL_TRANSACTION % { - 'transaction': render_section('Transaction', schema), - 'condition': render_section('Condition', defs['condition']), - 'fulfillment': render_section('Fulfillment', defs['fulfillment']), - 'asset': render_section('Asset', defs['asset']), - 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), - 'container': 'transaction-schema', - 'file': os.path.basename(__file__), - } - - write_schema_doc('transaction', doc) - - def write_schema_doc(name, doc): # Check base path exists base_path = os.path.join(os.path.dirname(__file__), 'source/schema') @@ -197,6 +220,7 @@ def resolve_ref(ref): def main(): """ Main function """ generate_transaction_docs() + generate_vote_docs() def setup(*_): diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index 1c2b4428..932f1951 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -16,5 +16,6 @@ BigchainDB Server Documentation topic-guides/index data-models/index schema/transaction + schema/vote release-notes appendices/index diff --git a/docs/server/source/schema/vote.rst b/docs/server/source/schema/vote.rst new file mode 100644 index 00000000..98bed941 --- /dev/null +++ b/docs/server/source/schema/vote.rst @@ -0,0 +1,118 @@ +.. This file was auto generated by generate_schema_documentation.py + +=========== +Vote Schema +=========== + +Vote +---- + +A Vote is an endorsement of a Block (identified by a hash) by +a node (identified by a public key). + +The outer Vote object contains the details of the vote being made +as well as the signature and identifying information of the node +passing the vote. + + +Vote.node_pubkey +^^^^^^^^^^^^^^^^ + +**type:** string + +Ed25519 public key identifying the voting node. + + + +Vote.signature +^^^^^^^^^^^^^^ + +**type:** string + +Ed25519 signature of the ``vote`` object. + + +Vote.vote +^^^^^^^^^ + +**type:** object + +Vote details to be signed. + + + + + +Vote Details +------------ + +Vote details to be signed. + + +Vote.previous_block +^^^^^^^^^^^^^^^^^^^ + +**type:** string + +Sha3 identifier of the block that preceeds the block being voted on. +The notion of a "previous" block is subject to vote. + + + +Vote.voting_for_block +^^^^^^^^^^^^^^^^^^^^^ + +**type:** string + +Sha3 identifier of the block being voted on. + + + +Vote.is_block_valid +^^^^^^^^^^^^^^^^^^^ + +**type:** boolean + +This field is ``true`` if the block was deemed valid by the node. + + + +Vote.invalid_reason +^^^^^^^^^^^^^^^^^^^ + +**type:** string or null + +Reason the block is voted invalid, or ``null``. + + + +Vote.timestamp +^^^^^^^^^^^^^^ + +**type:** string + +Unix timestamp that the vote was created by the node, according +to the system time of the node. + + + + + + +.. raw:: html + + From ecbeaa0b2517f7ffde1d98fd95abf96beb6e4c0e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 16:36:38 +0100 Subject: [PATCH 04/25] vote.id doesn't exist in server --- bigchaindb/db/backends/rethinkdb.py | 6 ++++-- docs/server/source/data-models/vote-model.md | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bigchaindb/db/backends/rethinkdb.py b/bigchaindb/db/backends/rethinkdb.py index 39d2aff9..55535c65 100644 --- a/bigchaindb/db/backends/rethinkdb.py +++ b/bigchaindb/db/backends/rethinkdb.py @@ -230,7 +230,8 @@ class RethinkDBBackend: """ return self.connection.run( r.table('votes', read_mode=self.read_mode) - .between([block_id, r.minval], [block_id, r.maxval], index='block_and_voter')) + .between([block_id, r.minval], [block_id, r.maxval], index='block_and_voter') + .without('id')) def get_votes_by_block_id_and_voter(self, block_id, node_pubkey): """Get all the votes casted for a specific block by a specific voter. @@ -244,7 +245,8 @@ class RethinkDBBackend: """ return self.connection.run( r.table('votes', read_mode=self.read_mode) - .get_all([block_id, node_pubkey], index='block_and_voter')) + .get_all([block_id, node_pubkey], index='block_and_voter') + .without('id')) def write_block(self, block, durability='soft'): """Write a block to the bigchain table. diff --git a/docs/server/source/data-models/vote-model.md b/docs/server/source/data-models/vote-model.md index 25d5029c..dc2b6ae4 100644 --- a/docs/server/source/data-models/vote-model.md +++ b/docs/server/source/data-models/vote-model.md @@ -4,7 +4,6 @@ A vote has the following structure: ```json { - "id": "", "node_pubkey": "", "vote": { "voting_for_block": "", From 855dc7a5e8885c26bb6a96842459ade2ffb2b137 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 16:37:26 +0100 Subject: [PATCH 05/25] refactor schema __init__ slightly --- bigchaindb/common/schema/__init__.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index a49495f9..89a6a820 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -23,15 +23,20 @@ def _validate_schema(schema, body): TX_SCHEMA_PATH, TX_SCHEMA = _load_schema('transaction') + + +def validate_transaction_schema(tx): + """ Validate a transaction dict """ + _validate_schema(TX_SCHEMA, tx) + + VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote') -def validate_transaction_schema(tx_body): - """ Validate a transaction dict """ - _validate_schema(TX_SCHEMA, tx_body) - - -def validate_vote_schema(tx_body): +def validate_vote_schema(vote): """ Validate a vote dict """ - _validate_schema(VOTE_SCHEMA, tx_body) - + # A vote does not have an ID, but the database may add one. + if 'id' in vote: + vote = dict(vote) + del vote['id'] + _validate_schema(VOTE_SCHEMA, vote) From c43bf101517500bd230f0ebe8ebfd931e7b8b70c Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 16:57:21 +0100 Subject: [PATCH 06/25] add call to vote schema validate in consensus.py --- bigchaindb/consensus.py | 11 +++++++++-- bigchaindb/core.py | 2 +- tests/pipelines/test_vote.py | 11 +++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index bfb054af..10024d76 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -1,4 +1,5 @@ from bigchaindb.util import verify_vote_signature +from bigchaindb.common.schema import validate_vote_schema, SchemaValidationError class BaseConsensusRules(): @@ -19,10 +20,16 @@ class BaseConsensusRules(): return block.validate(bigchain) @staticmethod - def verify_vote_signature(voters, signed_vote): + def verify_vote(voters, signed_vote): """Verify the signature of a vote. Refer to the documentation of :func:`bigchaindb.util.verify_signature`. """ - return verify_vote_signature(voters, signed_vote) + try: + validate_vote_schema(signed_vote) + except SchemaValidationError: + # TODO: log this. + return False + else: + return verify_vote_signature(voters, signed_vote) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 34dbab84..b59f4ebd 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -648,7 +648,7 @@ class Bigchain(object): prev_block = [vote['vote']['previous_block'] for vote in votes] # vote_validity checks whether a vote is valid # or invalid, e.g. [False, True, True] - vote_validity = [self.consensus.verify_vote_signature(voters, vote) for vote in votes] + vote_validity = [self.consensus.verify_vote(voters, vote) for vote in votes] # element-wise product of stated vote and validity of vote # vote_cast = [True, True, False] and diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index edbc5f4c..227a23a2 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -17,6 +17,9 @@ def dummy_block(b): return block +DUMMY_SHA3 = '0123456789abcdef' * 4 + + def test_vote_creation_valid(b): from bigchaindb.common import crypto from bigchaindb.common.util import serialize @@ -24,11 +27,11 @@ def test_vote_creation_valid(b): # create valid block block = dummy_block(b) # retrieve vote - vote = b.vote(block.id, 'abc', True) + vote = b.vote(block.id, DUMMY_SHA3, True) # assert vote is correct assert vote['vote']['voting_for_block'] == block.id - assert vote['vote']['previous_block'] == 'abc' + assert vote['vote']['previous_block'] == DUMMY_SHA3 assert vote['vote']['is_block_valid'] is True assert vote['vote']['invalid_reason'] is None assert vote['node_pubkey'] == b.me @@ -44,11 +47,11 @@ def test_vote_creation_invalid(b): # create valid block block = dummy_block(b) # retrieve vote - vote = b.vote(block.id, 'abc', False) + vote = b.vote(block.id, DUMMY_SHA3, False) # assert vote is correct assert vote['vote']['voting_for_block'] == block.id - assert vote['vote']['previous_block'] == 'abc' + assert vote['vote']['previous_block'] == DUMMY_SHA3 assert vote['vote']['is_block_valid'] is False assert vote['vote']['invalid_reason'] is None assert vote['node_pubkey'] == b.me From ef5f3ddd282952cefd60fd7481369974cc0842e7 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 11:26:04 +0100 Subject: [PATCH 07/25] better logging and arrangement for vote verification --- bigchaindb/consensus.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 10024d76..ae10b822 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -1,5 +1,11 @@ +import logging + from bigchaindb.util import verify_vote_signature -from bigchaindb.common.schema import validate_vote_schema, SchemaValidationError +from bigchaindb.common.schema import SchemaValidationError, \ + validate_vote_schema + + +logger = logging.getLogger(__name__) class BaseConsensusRules(): @@ -26,10 +32,12 @@ class BaseConsensusRules(): Refer to the documentation of :func:`bigchaindb.util.verify_signature`. """ - try: - validate_vote_schema(signed_vote) - except SchemaValidationError: - # TODO: log this. - return False - else: - return verify_vote_signature(voters, signed_vote) + if verify_vote_signature(voters, signed_vote): + try: + validate_vote_schema(signed_vote) + return True + except SchemaValidationError as exc: + logger.warning(exc) + logger.warning("Vote failed signature verification: " + "%s with voters: %s", signed_vote, voters) + return False From 44a43dcf949b48d8175b1b5382fa0a9cf42247d8 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 11:58:41 +0100 Subject: [PATCH 08/25] make all tests pass after vote schema introduction --- tests/db/test_bigchain_api.py | 6 +-- tests/pipelines/test_election.py | 30 ++++++----- tests/pipelines/test_vote.py | 85 ++++++++++++++++---------------- 3 files changed, 65 insertions(+), 56 deletions(-) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index afe2bbd6..9ad3732f 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -108,7 +108,7 @@ class TestBigchainApi(object): block2 = b.create_block([transfer_tx]) b.write_block(block2, durability='hard') - monkeypatch.setattr('time.time', lambda: 3) + monkeypatch.setattr('time.time', lambda: 3333333333) transfer_tx2 = Transaction.transfer(tx.to_inputs(), [([b.me], 1)], tx.asset) transfer_tx2 = transfer_tx2.sign([b.me_private]) @@ -137,7 +137,7 @@ class TestBigchainApi(object): block1 = b.create_block([tx]) b.write_block(block1, durability='hard') - monkeypatch.setattr('time.time', lambda: 2) + monkeypatch.setattr('time.time', lambda: 2222222222) block2 = b.create_block([tx]) b.write_block(block2, durability='hard') @@ -161,7 +161,7 @@ class TestBigchainApi(object): block1 = b.create_block([tx1]) b.write_block(block1, durability='hard') - monkeypatch.setattr('time.time', lambda: 2) + monkeypatch.setattr('time.time', lambda: 2222222222) tx2 = Transaction.create([b.me], [([b.me], 1)]) tx2 = tx2.sign([b.me_private]) block2 = b.create_block([tx2]) diff --git a/tests/pipelines/test_election.py b/tests/pipelines/test_election.py index 73c3b786..300dee3f 100644 --- a/tests/pipelines/test_election.py +++ b/tests/pipelines/test_election.py @@ -28,8 +28,10 @@ def test_check_for_quorum_invalid(b, user_pk): b.write_block(test_block) # split_vote (invalid) - votes = [member.vote(test_block.id, 'abc', True) for member in test_federation[:2]] + \ - [member.vote(test_block.id, 'abc', False) for member in test_federation[2:]] + votes = [member.vote(test_block.id, 'a' * 64, True) + for member in test_federation[:2]] + \ + [member.vote(test_block.id, 'b' * 64, False) + for member in test_federation[2:]] # cast votes for vote in votes: @@ -49,8 +51,10 @@ def test_check_for_quorum_invalid_prev_node(b, user_pk): # simulate a federation with four voters key_pairs = [crypto.generate_key_pair() for _ in range(4)] - test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) - for key_pair in key_pairs] + test_federation = [ + Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs + ] # add voters to block and write test_block.voters = [key_pair[1] for key_pair in key_pairs] @@ -58,8 +62,10 @@ def test_check_for_quorum_invalid_prev_node(b, user_pk): b.write_block(test_block) # split vote over prev node - votes = [member.vote(test_block.id, 'abc', True) for member in test_federation[:2]] + \ - [member.vote(test_block.id, 'def', True) for member in test_federation[2:]] + votes = [member.vote(test_block.id, 'a' * 64, True) + for member in test_federation[:2]] + \ + [member.vote(test_block.id, 'b' * 64, True) + for member in test_federation[2:]] # cast votes for vote in votes: @@ -80,8 +86,10 @@ def test_check_for_quorum_valid(b, user_pk): # simulate a federation with four voters key_pairs = [crypto.generate_key_pair() for _ in range(4)] - test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) - for key_pair in key_pairs] + test_federation = [ + Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs + ] # add voters to block and write test_block.voters = [key_pair[1] for key_pair in key_pairs] @@ -89,7 +97,7 @@ def test_check_for_quorum_valid(b, user_pk): b.write_block(test_block) # votes for block one - votes = [member.vote(test_block.id, 'abc', True) + votes = [member.vote(test_block.id, 'a' * 64, True) for member in test_federation] # cast votes for vote in votes: @@ -158,8 +166,8 @@ def test_full_pipeline(b, user_pk): pipeline.start() time.sleep(1) # vote one block valid, one invalid - vote_valid = b.vote(valid_block.id, 'abc', True) - vote_invalid = b.vote(invalid_block.id, 'abc', False) + vote_valid = b.vote(valid_block.id, 'b' * 64, True) + vote_invalid = b.vote(invalid_block.id, 'c' * 64, False) b.write_vote(vote_valid) b.write_vote(vote_invalid) diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index 227a23a2..be3652b2 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -37,7 +37,7 @@ def test_vote_creation_valid(b): assert vote['node_pubkey'] == b.me assert isinstance(vote['signature'], str) assert crypto.PublicKey(b.me).verify(serialize(vote['vote']).encode(), - vote['signature']) is True + vote['signature']) is True def test_vote_creation_invalid(b): @@ -56,7 +56,7 @@ def test_vote_creation_invalid(b): assert vote['vote']['invalid_reason'] is None assert vote['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialize(vote['vote']).encode(), - vote['signature']) is True + vote['signature']) is True def test_vote_ungroup_returns_a_set_of_results(b): @@ -160,7 +160,7 @@ def test_valid_block_voting_sequential(b, monkeypatch): from bigchaindb.common import crypto, util from bigchaindb.pipelines import vote - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) genesis = b.create_genesis_block() vote_obj = vote.Vote() block = dummy_block(b) @@ -176,12 +176,12 @@ def test_valid_block_voting_sequential(b, monkeypatch): 'previous_block': genesis.id, 'is_block_valid': True, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_valid_block_voting_multiprocessing(b, monkeypatch): @@ -191,7 +191,7 @@ def test_valid_block_voting_multiprocessing(b, monkeypatch): inpipe = Pipe() outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) genesis = b.create_genesis_block() vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) @@ -210,12 +210,12 @@ def test_valid_block_voting_multiprocessing(b, monkeypatch): 'previous_block': genesis.id, 'is_block_valid': True, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_valid_block_voting_with_create_transaction(b, monkeypatch): @@ -230,7 +230,7 @@ def test_valid_block_voting_with_create_transaction(b, monkeypatch): tx = Transaction.create([b.me], [([test_user_pub], 1)]) tx = tx.sign([b.me_private]) - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) block = b.create_block([tx]) inpipe = Pipe() @@ -251,12 +251,12 @@ def test_valid_block_voting_with_create_transaction(b, monkeypatch): 'previous_block': genesis.id, 'is_block_valid': True, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_valid_block_voting_with_transfer_transactions(monkeypatch, b): @@ -271,7 +271,7 @@ def test_valid_block_voting_with_transfer_transactions(monkeypatch, b): tx = Transaction.create([b.me], [([test_user_pub], 1)]) tx = tx.sign([b.me_private]) - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) block = b.create_block([tx]) b.write_block(block, durability='hard') @@ -281,7 +281,7 @@ def test_valid_block_voting_with_transfer_transactions(monkeypatch, b): tx.asset) tx2 = tx2.sign([test_user_priv]) - monkeypatch.setattr('time.time', lambda: 2) + monkeypatch.setattr('time.time', lambda: 2222222222) block2 = b.create_block([tx2]) b.write_block(block2, durability='hard') @@ -306,12 +306,12 @@ def test_valid_block_voting_with_transfer_transactions(monkeypatch, b): 'previous_block': genesis.id, 'is_block_valid': True, 'invalid_reason': None, - 'timestamp': '2'} + 'timestamp': '2222222222'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True vote2_rs = b.backend.get_votes_by_block_id_and_voter(block2.id, b.me) vote2_doc = vote2_rs.next() @@ -320,12 +320,12 @@ def test_valid_block_voting_with_transfer_transactions(monkeypatch, b): 'previous_block': block.id, 'is_block_valid': True, 'invalid_reason': None, - 'timestamp': '2'} + 'timestamp': '2222222222'} serialized_vote2 = util.serialize(vote2_doc['vote']).encode() assert vote2_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote2, - vote2_doc['signature']) is True + vote2_doc['signature']) is True def test_unsigned_tx_in_block_voting(monkeypatch, b, user_pk): @@ -336,7 +336,7 @@ def test_unsigned_tx_in_block_voting(monkeypatch, b, user_pk): inpipe = Pipe() outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) genesis = b.create_genesis_block() vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) @@ -357,12 +357,12 @@ def test_unsigned_tx_in_block_voting(monkeypatch, b, user_pk): 'previous_block': genesis.id, 'is_block_valid': False, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_invalid_id_tx_in_block_voting(monkeypatch, b, user_pk): @@ -373,7 +373,7 @@ def test_invalid_id_tx_in_block_voting(monkeypatch, b, user_pk): inpipe = Pipe() outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) genesis = b.create_genesis_block() vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) @@ -396,12 +396,12 @@ def test_invalid_id_tx_in_block_voting(monkeypatch, b, user_pk): 'previous_block': genesis.id, 'is_block_valid': False, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_invalid_content_in_tx_in_block_voting(monkeypatch, b, user_pk): @@ -412,7 +412,7 @@ def test_invalid_content_in_tx_in_block_voting(monkeypatch, b, user_pk): inpipe = Pipe() outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) genesis = b.create_genesis_block() vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) @@ -435,12 +435,12 @@ def test_invalid_content_in_tx_in_block_voting(monkeypatch, b, user_pk): 'previous_block': genesis.id, 'is_block_valid': False, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_invalid_block_voting(monkeypatch, b, user_pk): @@ -450,7 +450,7 @@ def test_invalid_block_voting(monkeypatch, b, user_pk): inpipe = Pipe() outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) genesis = b.create_genesis_block() vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) @@ -470,12 +470,12 @@ def test_invalid_block_voting(monkeypatch, b, user_pk): 'previous_block': genesis.id, 'is_block_valid': False, 'invalid_reason': None, - 'timestamp': '1'} + 'timestamp': '1111111111'} serialized_vote = util.serialize(vote_doc['vote']).encode() assert vote_doc['node_pubkey'] == b.me assert crypto.PublicKey(b.me).verify(serialized_vote, - vote_doc['signature']) is True + vote_doc['signature']) is True def test_voter_considers_unvoted_blocks_when_single_node(monkeypatch, b): @@ -483,17 +483,17 @@ def test_voter_considers_unvoted_blocks_when_single_node(monkeypatch, b): outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) b.create_genesis_block() block_ids = [] # insert blocks in the database while the voter process is not listening # (these blocks won't appear in the changefeed) - monkeypatch.setattr('time.time', lambda: 2) + monkeypatch.setattr('time.time', lambda: 2222222222) block_1 = dummy_block(b) block_ids.append(block_1.id) b.write_block(block_1, durability='hard') - monkeypatch.setattr('time.time', lambda: 3) + monkeypatch.setattr('time.time', lambda: 3333333333) block_2 = dummy_block(b) block_ids.append(block_2.id) b.write_block(block_2, durability='hard') @@ -508,7 +508,7 @@ def test_voter_considers_unvoted_blocks_when_single_node(monkeypatch, b): outpipe.get() # create a new block that will appear in the changefeed - monkeypatch.setattr('time.time', lambda: 4) + monkeypatch.setattr('time.time', lambda: 4444444444) block_3 = dummy_block(b) block_ids.append(block_3.id) b.write_block(block_3, durability='hard') @@ -530,16 +530,16 @@ def test_voter_chains_blocks_with_the_previous_ones(monkeypatch, b): outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) b.create_genesis_block() block_ids = [] - monkeypatch.setattr('time.time', lambda: 2) + monkeypatch.setattr('time.time', lambda: 2222222222) block_1 = dummy_block(b) block_ids.append(block_1.id) b.write_block(block_1, durability='hard') - monkeypatch.setattr('time.time', lambda: 3) + monkeypatch.setattr('time.time', lambda: 3333333333) block_2 = dummy_block(b) block_ids.append(block_2.id) b.write_block(block_2, durability='hard') @@ -556,12 +556,13 @@ def test_voter_chains_blocks_with_the_previous_ones(monkeypatch, b): # retrive blocks from bigchain blocks = [b.get_block(_id) for _id in block_ids] + # retrieve votes votes = [list(b.backend.get_votes_by_block_id(_id))[0] for _id in block_ids] - assert votes[0]['vote']['voting_for_block'] in (blocks[0]['id'], blocks[1]['id']) - assert votes[1]['vote']['voting_for_block'] in (blocks[0]['id'], blocks[1]['id']) + assert ({v['vote']['voting_for_block'] for v in votes} == + {block['id'] for block in blocks}) def test_voter_checks_for_previous_vote(monkeypatch, b): @@ -570,10 +571,10 @@ def test_voter_checks_for_previous_vote(monkeypatch, b): inpipe = Pipe() outpipe = Pipe() - monkeypatch.setattr('time.time', lambda: 1) + monkeypatch.setattr('time.time', lambda: 1111111111) b.create_genesis_block() - monkeypatch.setattr('time.time', lambda: 2) + monkeypatch.setattr('time.time', lambda: 2222222222) block_1 = dummy_block(b) inpipe.put(block_1.to_dict()) assert len(list(b.backend.get_votes_by_block_id(block_1.id))) == 0 @@ -586,11 +587,11 @@ def test_voter_checks_for_previous_vote(monkeypatch, b): outpipe.get() # queue block for voting AGAIN - monkeypatch.setattr('time.time', lambda: 3) + monkeypatch.setattr('time.time', lambda: 3333333333) inpipe.put(block_1.to_dict()) # queue another block - monkeypatch.setattr('time.time', lambda: 4) + monkeypatch.setattr('time.time', lambda: 4444444444) block_2 = dummy_block(b) inpipe.put(block_2.to_dict()) From bfb5be3ba44b93001ee178be45f06c8e041357b2 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 12:08:35 +0100 Subject: [PATCH 09/25] refactor schema tests into separate modules --- tests/common/schema/test_schema.py | 25 +++++++++++++++ .../test_transaction_schema.py} | 31 ++++--------------- tests/common/schema/test_vote_schema.py | 0 3 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 tests/common/schema/test_schema.py rename tests/common/{test_schema.py => schema/test_transaction_schema.py} (53%) create mode 100644 tests/common/schema/test_vote_schema.py diff --git a/tests/common/schema/test_schema.py b/tests/common/schema/test_schema.py new file mode 100644 index 00000000..3b8bc6fc --- /dev/null +++ b/tests/common/schema/test_schema.py @@ -0,0 +1,25 @@ +from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA + + +def _test_additionalproperties(node, path=''): + """ + Validate that each object node has additionalProperties set, so that + objects with junk keys do not pass as valid. + """ + if isinstance(node, list): + for i, nnode in enumerate(node): + _test_additionalproperties(nnode, path + str(i) + '.') + if isinstance(node, dict): + if node.get('type') == 'object': + assert 'additionalProperties' in node, \ + ("additionalProperties not set at path:" + path) + for name, val in node.items(): + _test_additionalproperties(val, path + name + '.') + + +def test_transaction_schema_additionalproperties(): + _test_additionalproperties(TX_SCHEMA) + + +def test_vote_schema_additionalproperties(): + _test_additionalproperties(VOTE_SCHEMA) diff --git a/tests/common/test_schema.py b/tests/common/schema/test_transaction_schema.py similarity index 53% rename from tests/common/test_schema.py rename to tests/common/schema/test_transaction_schema.py index 1827d3cb..c9545ab3 100644 --- a/tests/common/test_schema.py +++ b/tests/common/schema/test_transaction_schema.py @@ -1,7 +1,7 @@ from pytest import raises from bigchaindb.common.exceptions import SchemaValidationError -from bigchaindb.common.schema import TX_SCHEMA, validate_transaction_schema +from bigchaindb.common.schema import validate_transaction_schema def test_validate_transaction_create(create_tx): @@ -16,6 +16,11 @@ def test_validate_transaction_signed_transfer(signed_transfer_tx): validate_transaction_schema(signed_transfer_tx.to_dict()) +def test_validate_transaction_fails(): + with raises(SchemaValidationError): + validate_transaction_schema({}) + + def test_validate_fails_metadata_empty_dict(create_tx): create_tx.metadata = {'a': 1} validate_transaction_schema(create_tx.to_dict()) @@ -24,27 +29,3 @@ def test_validate_fails_metadata_empty_dict(create_tx): create_tx.metadata = {} with raises(SchemaValidationError): validate_transaction_schema(create_tx.to_dict()) - - -def test_validation_fails(): - with raises(SchemaValidationError): - validate_transaction_schema({}) - - -def test_addition_properties_always_set(): - """ - Validate that each object node has additionalProperties set, so that - transactions with junk keys do not pass as valid. - """ - def walk(node, path=''): - if isinstance(node, list): - for i, nnode in enumerate(node): - walk(nnode, path + str(i) + '.') - if isinstance(node, dict): - if node.get('type') == 'object': - assert 'additionalProperties' in node, \ - ("additionalProperties not set at path:" + path) - for name, val in node.items(): - walk(val, path + name + '.') - - walk(TX_SCHEMA) diff --git a/tests/common/schema/test_vote_schema.py b/tests/common/schema/test_vote_schema.py new file mode 100644 index 00000000..e69de29b From f38028d727feac61490b3430e76b3f366e50f390 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 12:13:56 +0100 Subject: [PATCH 10/25] basic tests for vote schema validator --- tests/common/schema/test_vote_schema.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/common/schema/test_vote_schema.py b/tests/common/schema/test_vote_schema.py index e69de29b..a6340c5e 100644 --- a/tests/common/schema/test_vote_schema.py +++ b/tests/common/schema/test_vote_schema.py @@ -0,0 +1,23 @@ +from pytest import raises + +from bigchaindb.common.exceptions import SchemaValidationError +from bigchaindb.common.schema import validate_vote_schema + + +def test_validate_vote(): + validate_vote_schema({ + 'node_pubkey': 'c' * 44, + 'signature': 'd' * 86, + 'vote': { + 'voting_for_block': 'a' * 64, + 'previous_block': 'b' * 64, + 'is_block_valid': False, + 'invalid_reason': None, + 'timestamp': '1111111111' + } + }) + + +def test_validate_vote_fails(): + with raises(SchemaValidationError): + validate_vote_schema({}) From f2e6e4d13d30b0cefd0b5ea7c09996a6d8261bf6 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 14:10:37 +0100 Subject: [PATCH 11/25] add tests to get coverage of consensus.py --- bigchaindb/common/schema/__init__.py | 4 --- bigchaindb/consensus.py | 5 ++-- tests/common/schema/test_vote_schema.py | 14 ++------- tests/conftest.py | 18 +++++++++++- tests/test_consensus.py | 39 +++++++++++++++++++++++-- 5 files changed, 59 insertions(+), 21 deletions(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 89a6a820..9f233dc9 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -35,8 +35,4 @@ VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote') def validate_vote_schema(vote): """ Validate a vote dict """ - # A vote does not have an ID, but the database may add one. - if 'id' in vote: - vote = dict(vote) - del vote['id'] _validate_schema(VOTE_SCHEMA, vote) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index ae10b822..a3e57863 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -38,6 +38,7 @@ class BaseConsensusRules(): return True except SchemaValidationError as exc: logger.warning(exc) - logger.warning("Vote failed signature verification: " - "%s with voters: %s", signed_vote, voters) + else: + logger.warning("Vote failed signature verification: " + "%s with voters: %s", signed_vote, voters) return False diff --git a/tests/common/schema/test_vote_schema.py b/tests/common/schema/test_vote_schema.py index a6340c5e..a9de9492 100644 --- a/tests/common/schema/test_vote_schema.py +++ b/tests/common/schema/test_vote_schema.py @@ -4,18 +4,8 @@ from bigchaindb.common.exceptions import SchemaValidationError from bigchaindb.common.schema import validate_vote_schema -def test_validate_vote(): - validate_vote_schema({ - 'node_pubkey': 'c' * 44, - 'signature': 'd' * 86, - 'vote': { - 'voting_for_block': 'a' * 64, - 'previous_block': 'b' * 64, - 'is_block_valid': False, - 'invalid_reason': None, - 'timestamp': '1111111111' - } - }) +def test_validate_vote(structurally_valid_vote): + validate_vote_schema(structurally_valid_vote) def test_validate_vote_fails(): diff --git a/tests/conftest.py b/tests/conftest.py index 957a5698..66305b6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,7 +38,8 @@ def ignore_local_config_file(monkeypatch): def mock_file_config(filename=None): raise FileNotFoundError() - monkeypatch.setattr('bigchaindb.config_utils.file_config', mock_file_config) + monkeypatch.setattr('bigchaindb.config_utils.file_config', + mock_file_config) @pytest.fixture(scope='function', autouse=True) @@ -86,3 +87,18 @@ def signed_transfer_tx(signed_create_tx, user_pk, user_sk): inputs = signed_create_tx.to_inputs() tx = Transaction.transfer(inputs, [([user_pk], 1)], signed_create_tx.asset) return tx.sign([user_sk]) + + +@pytest.fixture +def structurally_valid_vote(): + return { + 'node_pubkey': 'c' * 44, + 'signature': 'd' * 86, + 'vote': { + 'voting_for_block': 'a' * 64, + 'previous_block': 'b' * 64, + 'is_block_valid': False, + 'invalid_reason': None, + 'timestamp': '1111111111' + } + } diff --git a/tests/test_consensus.py b/tests/test_consensus.py index ef988726..43c4ef3e 100644 --- a/tests/test_consensus.py +++ b/tests/test_consensus.py @@ -1,2 +1,37 @@ -class TestBaseConsensusRules(object): - pass +from bigchaindb.consensus import BaseConsensusRules +from bigchaindb.common import crypto +from bigchaindb.common.util import serialize + + +def test_verify_vote_passes(b, structurally_valid_vote): + vote_body = structurally_valid_vote['vote'] + vote_data = serialize(vote_body) + signature = crypto.PrivateKey(b.me_private).sign(vote_data.encode()) + vote_signed = { + 'node_pubkey': b.me, + 'signature': signature.decode(), + 'vote': vote_body + } + assert BaseConsensusRules.verify_vote([b.me], vote_signed) + + +def test_verify_vote_fails_signature(b, structurally_valid_vote): + vote_body = structurally_valid_vote['vote'] + vote_signed = { + 'node_pubkey': b.me, + 'signature': 'a' * 86, + 'vote': vote_body + } + assert not BaseConsensusRules.verify_vote([b.me], vote_signed) + + +def test_verify_vote_fails_schema(b): + vote_body = {} + vote_data = serialize(vote_body) + signature = crypto.PrivateKey(b.me_private).sign(vote_data.encode()) + vote_signed = { + 'node_pubkey': b.me, + 'signature': signature.decode(), + 'vote': vote_body + } + assert not BaseConsensusRules.verify_vote([b.me], vote_signed) From 7453e57e7b98f3cfd6a6b3d4bb56325bcf6174a6 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 15:12:12 +0100 Subject: [PATCH 12/25] remove descriptions from yaml schemas loaded for validating in order to reduce noise in error descriptions --- bigchaindb/common/schema/__init__.py | 16 ++++++++++- tests/common/test_schema.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/common/test_schema.py diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 9f233dc9..f3ad9f30 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -7,11 +7,25 @@ import yaml from bigchaindb.common.exceptions import SchemaValidationError +def drop_schema_descriptions(node): + """ Drop descriptions from schema, since they clutter log output """ + if isinstance(node, list): + any(map(drop_schema_descriptions, node)) + elif isinstance(node, dict): + print(node) + if node.get('type') == 'object': + if 'description' in node: + del node['description'] + any(map(drop_schema_descriptions, node.values())) + + def _load_schema(name): """ Load a schema from disk """ path = os.path.join(os.path.dirname(__file__), name + '.yaml') with open(path) as handle: - return path, yaml.safe_load(handle) + schema = yaml.safe_load(handle) + drop_schema_descriptions(schema) + return path, schema def _validate_schema(schema, body): diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py new file mode 100644 index 00000000..8f611554 --- /dev/null +++ b/tests/common/test_schema.py @@ -0,0 +1,42 @@ +from copy import deepcopy +from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA, \ + drop_schema_descriptions + + +def _test_additionalproperties(node, path=''): + """ + Validate that each object node has additionalProperties set, so that + objects with junk keys do not pass as valid. + """ + if isinstance(node, list): + for i, nnode in enumerate(node): + _test_additionalproperties(nnode, path + str(i) + '.') + if isinstance(node, dict): + if node.get('type') == 'object': + assert 'additionalProperties' in node, \ + ("additionalProperties not set at path:" + path) + for name, val in node.items(): + _test_additionalproperties(val, path + name + '.') + + +def test_transaction_schema_additionalproperties(): + _test_additionalproperties(TX_SCHEMA) + + +def test_vote_schema_additionalproperties(): + _test_additionalproperties(VOTE_SCHEMA) + + +def test_drop_descriptions(): + node = { + 'a': 1, + 'description': 'abc', + 'b': [{ + 'type': 'object', + 'description': 'gone, baby', + }] + } + node2 = deepcopy(node) + drop_schema_descriptions(node) + del node2['b'][0]['description'] + assert node == node2 From 0f824b071f0fd2878b56d2dcd6d399a7e6ebe582 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 15:37:18 +0100 Subject: [PATCH 13/25] fix unclear code in drop_schema_descriptions and remove leftover print() --- bigchaindb/common/schema/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index f3ad9f30..96b2c8be 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -10,13 +10,14 @@ from bigchaindb.common.exceptions import SchemaValidationError def drop_schema_descriptions(node): """ Drop descriptions from schema, since they clutter log output """ if isinstance(node, list): - any(map(drop_schema_descriptions, node)) + for val in node: + drop_schema_descriptions(val) elif isinstance(node, dict): - print(node) if node.get('type') == 'object': if 'description' in node: del node['description'] - any(map(drop_schema_descriptions, node.values())) + for val in node.values(): + drop_schema_descriptions(val) def _load_schema(name): From e861353a7363b750b685790af671f8bf16b43a7d Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 25 Nov 2016 18:47:36 +0100 Subject: [PATCH 14/25] documentation fixes for vote schema and remove vote-model.md in place of vote schema doc, and link to vote schema doc from data-models index --- bigchaindb/common/schema/vote.yaml | 12 ++++++---- docs/server/generate_schema_documentation.py | 9 ++++++++ docs/server/source/data-models/index.rst | 2 +- docs/server/source/data-models/vote-model.md | 19 ---------------- docs/server/source/schema/vote.rst | 23 +++++++++++++++----- 5 files changed, 36 insertions(+), 29 deletions(-) delete mode 100644 docs/server/source/data-models/vote-model.md diff --git a/bigchaindb/common/schema/vote.yaml b/bigchaindb/common/schema/vote.yaml index 8a09a10a..1cd602a1 100644 --- a/bigchaindb/common/schema/vote.yaml +++ b/bigchaindb/common/schema/vote.yaml @@ -25,12 +25,12 @@ properties: type: "string" pattern: "[1-9a-zA-Z^OIl]{86,88}" description: - Ed25519 signature of the ``vote`` object. + Ed25519 signature of the `Vote Details`_ object. vote: type: "object" additionalProperties: false description: | - Vote details to be signed. + `Vote Details`_ to be signed. required: - invalid_reason - is_block_valid @@ -41,12 +41,12 @@ properties: previous_block: "$ref": "#/definitions/sha3_hexdigest" description: | - Sha3 identifier of the block that preceeds the block being voted on. + SHA3 identifier of the block that preceeds the block being voted on. The notion of a "previous" block is subject to vote. voting_for_block: "$ref": "#/definitions/sha3_hexdigest" description: | - Sha3 identifier of the block being voted on. + SHA3 identifier of the block being voted on. is_block_valid: type: "boolean" description: | @@ -56,6 +56,10 @@ properties: - type: "string" description: | Reason the block is voted invalid, or ``null``. + + .. container:: notice + + **Note**: The invalid_reason was not being used and may be dropped in a future version of BigchainDB. See Issue `#217 `_ on GitHub. - type: "null" timestamp: type: "string" diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 704157f3..2c850580 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -35,6 +35,15 @@ TPL_STYLES = """ font-size: 100%%; font-family: monospace; } + .document .section p { + margin-bottom: 16px; + } + .notice { + margin: 0px 16px 16px 16px; + background-color: white; + border: 1px solid gold; + padding: 3px 6px; + } """ diff --git a/docs/server/source/data-models/index.rst b/docs/server/source/data-models/index.rst index 70a4fae4..86b62121 100644 --- a/docs/server/source/data-models/index.rst +++ b/docs/server/source/data-models/index.rst @@ -16,4 +16,4 @@ This section unpacks each one in turn. asset-model crypto-conditions block-model - vote-model + The Vote Model <../schema/vote> diff --git a/docs/server/source/data-models/vote-model.md b/docs/server/source/data-models/vote-model.md deleted file mode 100644 index dc2b6ae4..00000000 --- a/docs/server/source/data-models/vote-model.md +++ /dev/null @@ -1,19 +0,0 @@ -# The Vote Model - -A vote has the following structure: - -```json -{ - "node_pubkey": "", - "vote": { - "voting_for_block": "", - "previous_block": "", - "is_block_valid": "", - "invalid_reason": "" - }, - "signature": "" -} -``` - -Note: The `invalid_reason` was not being used and may be dropped in a future version of BigchainDB. See [Issue #217](https://github.com/bigchaindb/bigchaindb/issues/217) on GitHub. diff --git a/docs/server/source/schema/vote.rst b/docs/server/source/schema/vote.rst index 98bed941..312e1973 100644 --- a/docs/server/source/schema/vote.rst +++ b/docs/server/source/schema/vote.rst @@ -29,7 +29,7 @@ Vote.signature **type:** string -Ed25519 signature of the ``vote`` object. +Ed25519 signature of the `Vote Details`_ object. Vote.vote @@ -37,7 +37,7 @@ Vote.vote **type:** object -Vote details to be signed. +`Vote Details`_ to be signed. @@ -46,7 +46,7 @@ Vote details to be signed. Vote Details ------------ -Vote details to be signed. +`Vote Details`_ to be signed. Vote.previous_block @@ -54,7 +54,7 @@ Vote.previous_block **type:** string -Sha3 identifier of the block that preceeds the block being voted on. +SHA3 identifier of the block that preceeds the block being voted on. The notion of a "previous" block is subject to vote. @@ -64,7 +64,7 @@ Vote.voting_for_block **type:** string -Sha3 identifier of the block being voted on. +SHA3 identifier of the block being voted on. @@ -84,6 +84,10 @@ Vote.invalid_reason Reason the block is voted invalid, or ``null``. +.. container:: notice + + **Note**: The invalid_reason was not being used and may be dropped in a future version of BigchainDB. See Issue `#217 `_ on GitHub. + Vote.timestamp @@ -115,4 +119,13 @@ to the system time of the node. font-size: 100%; font-family: monospace; } + .document .section p { + margin-bottom: 16px; + } + .notice { + margin: 0px 16px 16px 16px; + background-color: white; + border: 1px solid gold; + padding: 3px 6px; + } From d7e0009ce528dd54be3ebc84e9ab5b3119dd69e7 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 28 Nov 2016 16:03:11 +0100 Subject: [PATCH 15/25] Drop definitions from schemas since they clutter log output --- bigchaindb/common/schema/__init__.py | 23 +++++++------- tests/common/test_schema.py | 45 ++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 96b2c8be..675b17e7 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -9,15 +9,16 @@ from bigchaindb.common.exceptions import SchemaValidationError def drop_schema_descriptions(node): """ Drop descriptions from schema, since they clutter log output """ - if isinstance(node, list): - for val in node: - drop_schema_descriptions(val) - elif isinstance(node, dict): - if node.get('type') == 'object': - if 'description' in node: - del node['description'] - for val in node.values(): - drop_schema_descriptions(val) + if 'description' in node: + del node['description'] + for n in node.get('properties', {}).values(): + drop_schema_descriptions(n) + for n in node.get('definitions', {}).values(): + drop_schema_descriptions(n) + for n in node.get('anyOf', []): + drop_schema_descriptions(n) + + def _load_schema(name): @@ -25,8 +26,8 @@ def _load_schema(name): path = os.path.join(os.path.dirname(__file__), name + '.yaml') with open(path) as handle: schema = yaml.safe_load(handle) - drop_schema_descriptions(schema) - return path, schema + drop_schema_descriptions(schema) + return path, schema def _validate_schema(schema, body): diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py index 8f611554..c7990c39 100644 --- a/tests/common/test_schema.py +++ b/tests/common/test_schema.py @@ -1,4 +1,3 @@ -from copy import deepcopy from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA, \ drop_schema_descriptions @@ -29,14 +28,42 @@ def test_vote_schema_additionalproperties(): def test_drop_descriptions(): node = { - 'a': 1, 'description': 'abc', - 'b': [{ - 'type': 'object', - 'description': 'gone, baby', - }] + 'properties': { + 'description': { + 'description': ('The property named "description" should stay' + 'but description meta field goes'), + }, + 'properties': { + 'description': 'this must go' + }, + 'any': { + 'anyOf': [ + { + 'description': 'must go' + } + ] + } + }, + 'definitions': { + 'wat': { + 'description': "go" + } + } } - node2 = deepcopy(node) drop_schema_descriptions(node) - del node2['b'][0]['description'] - assert node == node2 + expected = { + 'properties': { + 'description': {}, + 'properties': {}, + 'any': { + 'anyOf': [ + {} + ] + } + }, + 'definitions': { + 'wat': {}, + } + } + assert node == expected From 6208a93673cf725db26e5ba119332866a19b8eda Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 29 Nov 2016 17:02:56 +0100 Subject: [PATCH 16/25] fix linting issue --- bigchaindb/common/schema/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 675b17e7..f2feb22f 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -19,8 +19,6 @@ def drop_schema_descriptions(node): drop_schema_descriptions(n) - - def _load_schema(name): """ Load a schema from disk """ path = os.path.join(os.path.dirname(__file__), name + '.yaml') From a690fbee539be6294afadc31bef57df908b82f65 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 29 Nov 2016 17:42:16 +0100 Subject: [PATCH 17/25] documentation fix for vote schema --- bigchaindb/common/schema/vote.yaml | 4 ++-- docs/server/source/schema/vote.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigchaindb/common/schema/vote.yaml b/bigchaindb/common/schema/vote.yaml index 1cd602a1..49e5ae98 100644 --- a/bigchaindb/common/schema/vote.yaml +++ b/bigchaindb/common/schema/vote.yaml @@ -41,12 +41,12 @@ properties: previous_block: "$ref": "#/definitions/sha3_hexdigest" description: | - SHA3 identifier of the block that preceeds the block being voted on. + ID (SHA3 hash) of the block that precedes the block being voted on. The notion of a "previous" block is subject to vote. voting_for_block: "$ref": "#/definitions/sha3_hexdigest" description: | - SHA3 identifier of the block being voted on. + ID (SHA3 hash) of the block being voted on. is_block_valid: type: "boolean" description: | diff --git a/docs/server/source/schema/vote.rst b/docs/server/source/schema/vote.rst index 312e1973..292cec5c 100644 --- a/docs/server/source/schema/vote.rst +++ b/docs/server/source/schema/vote.rst @@ -54,7 +54,7 @@ Vote.previous_block **type:** string -SHA3 identifier of the block that preceeds the block being voted on. +ID (SHA3 hash) of the block that precedes the block being voted on. The notion of a "previous" block is subject to vote. @@ -64,7 +64,7 @@ Vote.voting_for_block **type:** string -SHA3 identifier of the block being voted on. +ID (SHA3 hash) of the block being voted on. From 7cb6c8913b94fe2ef57a6f466d2ddbcd2846432c Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 1 Dec 2016 15:55:24 +0100 Subject: [PATCH 18/25] unversion vote.rst since it's JITted now --- docs/server/source/schema/vote.rst | 131 ----------------------------- 1 file changed, 131 deletions(-) delete mode 100644 docs/server/source/schema/vote.rst diff --git a/docs/server/source/schema/vote.rst b/docs/server/source/schema/vote.rst deleted file mode 100644 index 292cec5c..00000000 --- a/docs/server/source/schema/vote.rst +++ /dev/null @@ -1,131 +0,0 @@ -.. This file was auto generated by generate_schema_documentation.py - -=========== -Vote Schema -=========== - -Vote ----- - -A Vote is an endorsement of a Block (identified by a hash) by -a node (identified by a public key). - -The outer Vote object contains the details of the vote being made -as well as the signature and identifying information of the node -passing the vote. - - -Vote.node_pubkey -^^^^^^^^^^^^^^^^ - -**type:** string - -Ed25519 public key identifying the voting node. - - - -Vote.signature -^^^^^^^^^^^^^^ - -**type:** string - -Ed25519 signature of the `Vote Details`_ object. - - -Vote.vote -^^^^^^^^^ - -**type:** object - -`Vote Details`_ to be signed. - - - - - -Vote Details ------------- - -`Vote Details`_ to be signed. - - -Vote.previous_block -^^^^^^^^^^^^^^^^^^^ - -**type:** string - -ID (SHA3 hash) of the block that precedes the block being voted on. -The notion of a "previous" block is subject to vote. - - - -Vote.voting_for_block -^^^^^^^^^^^^^^^^^^^^^ - -**type:** string - -ID (SHA3 hash) of the block being voted on. - - - -Vote.is_block_valid -^^^^^^^^^^^^^^^^^^^ - -**type:** boolean - -This field is ``true`` if the block was deemed valid by the node. - - - -Vote.invalid_reason -^^^^^^^^^^^^^^^^^^^ - -**type:** string or null - -Reason the block is voted invalid, or ``null``. - -.. container:: notice - - **Note**: The invalid_reason was not being used and may be dropped in a future version of BigchainDB. See Issue `#217 `_ on GitHub. - - - -Vote.timestamp -^^^^^^^^^^^^^^ - -**type:** string - -Unix timestamp that the vote was created by the node, according -to the system time of the node. - - - - - - -.. raw:: html - - From 624a0c972a4794854b926ffc5b397dd5cabf0474 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 1 Dec 2016 16:00:09 +0100 Subject: [PATCH 19/25] remove misplaced docs/generate --- docs/generate/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/generate/__init__.py diff --git a/docs/generate/__init__.py b/docs/generate/__init__.py deleted file mode 100644 index e69de29b..00000000 From 94df6fc00a4c0450dd90c045a81b96931dfee68b Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 1 Dec 2016 16:07:31 +0100 Subject: [PATCH 20/25] merge test_schema tests and tweak Data Models documentation index --- docs/server/source/data-models/index.rst | 2 +- tests/common/schema/test_schema.py | 46 +++++++++++++++- tests/common/test_schema.py | 69 ------------------------ 3 files changed, 46 insertions(+), 71 deletions(-) delete mode 100644 tests/common/test_schema.py diff --git a/docs/server/source/data-models/index.rst b/docs/server/source/data-models/index.rst index 86b62121..6fefe9d4 100644 --- a/docs/server/source/data-models/index.rst +++ b/docs/server/source/data-models/index.rst @@ -16,4 +16,4 @@ This section unpacks each one in turn. asset-model crypto-conditions block-model - The Vote Model <../schema/vote> + ../schema/vote diff --git a/tests/common/schema/test_schema.py b/tests/common/schema/test_schema.py index 3b8bc6fc..c7990c39 100644 --- a/tests/common/schema/test_schema.py +++ b/tests/common/schema/test_schema.py @@ -1,4 +1,5 @@ -from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA +from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA, \ + drop_schema_descriptions def _test_additionalproperties(node, path=''): @@ -23,3 +24,46 @@ def test_transaction_schema_additionalproperties(): def test_vote_schema_additionalproperties(): _test_additionalproperties(VOTE_SCHEMA) + + +def test_drop_descriptions(): + node = { + 'description': 'abc', + 'properties': { + 'description': { + 'description': ('The property named "description" should stay' + 'but description meta field goes'), + }, + 'properties': { + 'description': 'this must go' + }, + 'any': { + 'anyOf': [ + { + 'description': 'must go' + } + ] + } + }, + 'definitions': { + 'wat': { + 'description': "go" + } + } + } + drop_schema_descriptions(node) + expected = { + 'properties': { + 'description': {}, + 'properties': {}, + 'any': { + 'anyOf': [ + {} + ] + } + }, + 'definitions': { + 'wat': {}, + } + } + assert node == expected diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py deleted file mode 100644 index c7990c39..00000000 --- a/tests/common/test_schema.py +++ /dev/null @@ -1,69 +0,0 @@ -from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA, \ - drop_schema_descriptions - - -def _test_additionalproperties(node, path=''): - """ - Validate that each object node has additionalProperties set, so that - objects with junk keys do not pass as valid. - """ - if isinstance(node, list): - for i, nnode in enumerate(node): - _test_additionalproperties(nnode, path + str(i) + '.') - if isinstance(node, dict): - if node.get('type') == 'object': - assert 'additionalProperties' in node, \ - ("additionalProperties not set at path:" + path) - for name, val in node.items(): - _test_additionalproperties(val, path + name + '.') - - -def test_transaction_schema_additionalproperties(): - _test_additionalproperties(TX_SCHEMA) - - -def test_vote_schema_additionalproperties(): - _test_additionalproperties(VOTE_SCHEMA) - - -def test_drop_descriptions(): - node = { - 'description': 'abc', - 'properties': { - 'description': { - 'description': ('The property named "description" should stay' - 'but description meta field goes'), - }, - 'properties': { - 'description': 'this must go' - }, - 'any': { - 'anyOf': [ - { - 'description': 'must go' - } - ] - } - }, - 'definitions': { - 'wat': { - 'description': "go" - } - } - } - drop_schema_descriptions(node) - expected = { - 'properties': { - 'description': {}, - 'properties': {}, - 'any': { - 'anyOf': [ - {} - ] - } - }, - 'definitions': { - 'wat': {}, - } - } - assert node == expected From d9bc90e5aaf33dd24beb690dece0698bc662df42 Mon Sep 17 00:00:00 2001 From: libscott Date: Wed, 7 Dec 2016 15:06:49 +0100 Subject: [PATCH 21/25] Imports inside tests in test_consensus.py --- tests/test_consensus.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_consensus.py b/tests/test_consensus.py index 43c4ef3e..c5d7f7c1 100644 --- a/tests/test_consensus.py +++ b/tests/test_consensus.py @@ -1,9 +1,8 @@ -from bigchaindb.consensus import BaseConsensusRules -from bigchaindb.common import crypto -from bigchaindb.common.util import serialize - def test_verify_vote_passes(b, structurally_valid_vote): + from bigchaindb.consensus import BaseConsensusRules + from bigchaindb.common import crypto + from bigchaindb.common.util import serialize vote_body = structurally_valid_vote['vote'] vote_data = serialize(vote_body) signature = crypto.PrivateKey(b.me_private).sign(vote_data.encode()) @@ -16,6 +15,7 @@ def test_verify_vote_passes(b, structurally_valid_vote): def test_verify_vote_fails_signature(b, structurally_valid_vote): + from bigchaindb.consensus import BaseConsensusRules vote_body = structurally_valid_vote['vote'] vote_signed = { 'node_pubkey': b.me, @@ -26,6 +26,9 @@ def test_verify_vote_fails_signature(b, structurally_valid_vote): def test_verify_vote_fails_schema(b): + from bigchaindb.consensus import BaseConsensusRules + from bigchaindb.common import crypto + from bigchaindb.common.util import serialize vote_body = {} vote_data = serialize(vote_body) signature = crypto.PrivateKey(b.me_private).sign(vote_data.encode()) From ea6ae4db202fddf56f6dc45855c593bceccceed8 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 9 Dec 2016 09:49:27 +0100 Subject: [PATCH 22/25] revert data-models/vote-model documentation --- docs/server/source/data-models/index.rst | 2 +- docs/server/source/data-models/vote-model.md | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 docs/server/source/data-models/vote-model.md diff --git a/docs/server/source/data-models/index.rst b/docs/server/source/data-models/index.rst index 6fefe9d4..70a4fae4 100644 --- a/docs/server/source/data-models/index.rst +++ b/docs/server/source/data-models/index.rst @@ -16,4 +16,4 @@ This section unpacks each one in turn. asset-model crypto-conditions block-model - ../schema/vote + vote-model diff --git a/docs/server/source/data-models/vote-model.md b/docs/server/source/data-models/vote-model.md new file mode 100644 index 00000000..25d5029c --- /dev/null +++ b/docs/server/source/data-models/vote-model.md @@ -0,0 +1,20 @@ +# The Vote Model + +A vote has the following structure: + +```json +{ + "id": "", + "node_pubkey": "", + "vote": { + "voting_for_block": "", + "previous_block": "", + "is_block_valid": "", + "invalid_reason": "" + }, + "signature": "" +} +``` + +Note: The `invalid_reason` was not being used and may be dropped in a future version of BigchainDB. See [Issue #217](https://github.com/bigchaindb/bigchaindb/issues/217) on GitHub. From 4f83fce39d2bcfbe3646cccd93fa4271de76d90e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 9 Dec 2016 10:00:42 +0100 Subject: [PATCH 23/25] address some PR comments on coding style --- bigchaindb/consensus.py | 4 ++-- tests/common/schema/test_schema.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index a3e57863..6767ce6d 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -1,8 +1,8 @@ import logging from bigchaindb.util import verify_vote_signature -from bigchaindb.common.schema import SchemaValidationError, \ - validate_vote_schema +from bigchaindb.common.schema import (SchemaValidationError, + validate_vote_schema) logger = logging.getLogger(__name__) diff --git a/tests/common/schema/test_schema.py b/tests/common/schema/test_schema.py index c7990c39..0bb89eb4 100644 --- a/tests/common/schema/test_schema.py +++ b/tests/common/schema/test_schema.py @@ -1,5 +1,5 @@ -from bigchaindb.common.schema import TX_SCHEMA, VOTE_SCHEMA, \ - drop_schema_descriptions +from bigchaindb.common.schema import (TX_SCHEMA, VOTE_SCHEMA, + drop_schema_descriptions) def _test_additionalproperties(node, path=''): @@ -51,7 +51,6 @@ def test_drop_descriptions(): } } } - drop_schema_descriptions(node) expected = { 'properties': { 'description': {}, @@ -66,4 +65,5 @@ def test_drop_descriptions(): 'wat': {}, } } + drop_schema_descriptions(node) assert node == expected From a635838c588288dc2ae3c59b0802eb30adb31ed9 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 9 Dec 2016 11:34:42 +0100 Subject: [PATCH 24/25] few style changes for vote-schema.py --- bigchaindb/common/schema/__init__.py | 10 ++++------ tests/common/schema/test_schema.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index f2feb22f..52c70c13 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -28,6 +28,10 @@ def _load_schema(name): return path, schema +TX_SCHEMA_PATH, TX_SCHEMA = _load_schema('transaction') +VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote') + + def _validate_schema(schema, body): """ Validate data against a schema """ try: @@ -36,17 +40,11 @@ def _validate_schema(schema, body): raise SchemaValidationError(str(exc)) from exc -TX_SCHEMA_PATH, TX_SCHEMA = _load_schema('transaction') - - def validate_transaction_schema(tx): """ Validate a transaction dict """ _validate_schema(TX_SCHEMA, tx) -VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote') - - def validate_vote_schema(vote): """ Validate a vote dict """ _validate_schema(VOTE_SCHEMA, vote) diff --git a/tests/common/schema/test_schema.py b/tests/common/schema/test_schema.py index 0bb89eb4..a7cc6891 100644 --- a/tests/common/schema/test_schema.py +++ b/tests/common/schema/test_schema.py @@ -1,5 +1,5 @@ -from bigchaindb.common.schema import (TX_SCHEMA, VOTE_SCHEMA, - drop_schema_descriptions) +from bigchaindb.common.schema import ( + TX_SCHEMA, VOTE_SCHEMA, drop_schema_descriptions) def _test_additionalproperties(node, path=''): From 98ca6ee5ab909a7f038b118cdf2c3b1d0d2c7e0e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 9 Dec 2016 13:44:19 +0100 Subject: [PATCH 25/25] always try to include all operands on the first level of the statement --- docs/server/generate_schema_documentation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 2c850580..6b638d0a 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -48,7 +48,7 @@ TPL_STYLES = """ """ -TPL_TRANSACTION = """\ +TPL_TRANSACTION = TPL_STYLES + """\ .. This file was auto generated by %(file)s ================== @@ -90,7 +90,7 @@ Metadata -------- %(metadata)s -""" + TPL_STYLES +""" def generate_transaction_docs(): @@ -110,7 +110,7 @@ def generate_transaction_docs(): write_schema_doc('transaction', doc) -TPL_VOTE = """\ +TPL_VOTE = TPL_STYLES + """\ .. This file was auto generated by %(file)s =========== @@ -127,7 +127,7 @@ Vote Details %(vote_details)s -""" + TPL_STYLES +""" def generate_vote_docs():