From b7f70befe6cd6d8647aa2eaa3e9505348be8cf0e Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 1 Feb 2017 15:10:17 +0100 Subject: [PATCH 1/2] fix wrong logic in validate_block --- bigchaindb/models.py | 5 ----- tests/test_models.py | 13 ------------- 2 files changed, 18 deletions(-) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c6e81956..c3683a03 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -202,11 +202,6 @@ class Block(object): OperationError: If a non-federation node signed the Block. InvalidSignature: If a Block's signature is invalid. """ - - # First, make sure this node hasn't already voted on this block - if bigchain.has_previous_vote(self.id, self.voters): - return self - # Check if the block was created by a federation node possible_voters = (bigchain.nodes_except_me + [bigchain.me]) if self.node_pubkey not in possible_voters: diff --git a/tests/test_models.py b/tests/test_models.py index 7ab97e9e..58aa64fd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -163,16 +163,3 @@ class TestBlockModel(object): public_key = PublicKey(b.me) assert public_key.verify(expected_block_serialized, block.signature) - - def test_validate_already_voted_on_block(self, b, monkeypatch): - from unittest.mock import Mock - from bigchaindb.models import Transaction - - tx = Transaction.create([b.me], [([b.me], 1)]) - block = b.create_block([tx]) - - has_previous_vote = Mock() - has_previous_vote.return_value = True - monkeypatch.setattr(b, 'has_previous_vote', has_previous_vote) - assert block == block.validate(b) - assert has_previous_vote.called is True From 6fd8c7a20bdcf9295b0ec63ef14ed3cb67f49868 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 2 Feb 2017 09:45:13 +0100 Subject: [PATCH 2/2] Feat/105/secondary indexes inputs outputs (#1125) * Added inputs/outputs secondary indexes for rethinkdb Added tests. * Added inputs/outputs secondary indexes for mongodb Fixed tests. * fixed comment --- bigchaindb/backend/mongodb/query.py | 11 ++++++----- bigchaindb/backend/mongodb/schema.py | 12 ++++++++++++ bigchaindb/backend/rethinkdb/query.py | 15 ++++++++------- bigchaindb/backend/rethinkdb/schema.py | 25 +++++++++++++++++++++++++ tests/backend/mongodb/test_schema.py | 8 ++++---- tests/backend/rethinkdb/test_schema.py | 4 ++++ 6 files changed, 59 insertions(+), 16 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index d7ee6afc..e3b71315 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -143,6 +143,10 @@ def get_asset_by_id(conn, asset_id): @register_query(MongoDBConnection) def get_spent(conn, transaction_id, output): cursor = conn.db['bigchain'].aggregate([ + {'$match': { + 'block.transactions.inputs.fulfills.txid': transaction_id, + 'block.transactions.inputs.fulfills.output': output + }}, {'$unwind': '$block.transactions'}, {'$match': { 'block.transactions.inputs.fulfills.txid': transaction_id, @@ -157,12 +161,9 @@ def get_spent(conn, transaction_id, output): @register_query(MongoDBConnection) def get_owned_ids(conn, owner): cursor = conn.db['bigchain'].aggregate([ + {'$match': {'block.transactions.outputs.public_keys': owner}}, {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.outputs.public_keys': { - '$elemMatch': {'$eq': owner} - } - }} + {'$match': {'block.transactions.outputs.public_keys': owner}} ]) # we need to access some nested fields before returning so lets use a # generator to avoid having to read all records on the cursor at this point diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 2c526e7c..95c2d02a 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -63,6 +63,18 @@ def create_bigchain_secondary_index(conn, dbname): .create_index('block.transactions.transaction.asset.id', name='asset_id') + # secondary index on the public keys of outputs + conn.conn[dbname]['bigchain']\ + .create_index('block.transactions.outputs.public_keys', + name='outputs') + + # secondary index on inputs/transaction links (txid, output) + conn.conn[dbname]['bigchain']\ + .create_index([ + ('block.transactions.inputs.fulfills.txid', ASCENDING), + ('block.transactions.inputs.fulfills.output', ASCENDING), + ], name='inputs') + def create_backlog_secondary_index(conn, dbname): logger.info('Create `backlog` secondary index.') diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index aa7c3be6..99346984 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -111,21 +111,22 @@ def _get_asset_create_tx_query(asset_id): @register_query(RethinkDBConnection) def get_spent(connection, transaction_id, output): - # TODO: use index! return connection.run( r.table('bigchain', read_mode=READ_MODE) - .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda transaction: transaction['inputs'].contains( - lambda input: input['fulfills'] == {'txid': transaction_id, 'output': output}))) + .get_all([transaction_id, output], index='inputs') + .concat_map(lambda doc: doc['block']['transactions']) + .filter(lambda transaction: transaction['inputs'].contains( + lambda input_: input_['fulfills'] == {'txid': transaction_id, 'output': output}))) @register_query(RethinkDBConnection) def get_owned_ids(connection, owner): - # TODO: use index! return connection.run( r.table('bigchain', read_mode=READ_MODE) - .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda tx: tx['outputs'].contains( + .get_all(owner, index='outputs') + .distinct() + .concat_map(lambda doc: doc['block']['transactions']) + .filter(lambda tx: tx['outputs'].contains( lambda c: c['public_keys'].contains(owner)))) diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index 4a76a06b..997ec5fc 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -66,6 +66,31 @@ def create_bigchain_secondary_index(connection, dbname): .table('bigchain') .index_create('asset_id', r.row['block']['transactions']['asset']['id'], multi=True)) + # secondary index on the public keys of outputs + # the last reduce operation is to return a flatten list of public_keys + # without it we would need to match exactly the public_keys list. + # For instance querying for `pk1` would not match documents with + # `public_keys: [pk1, pk2, pk3]` + connection.run( + r.db(dbname) + .table('bigchain') + .index_create('outputs', + r.row['block']['transactions'] + .concat_map(lambda tx: tx['outputs']['public_keys']) + .reduce(lambda l, r: l + r), multi=True)) + + # secondary index on inputs/transaction links (txid, output) + connection.run( + r.db(dbname) + .table('bigchain') + .index_create('inputs', + r.row['block']['transactions'] + .concat_map(lambda tx: tx['inputs']['fulfills']) + .with_fields('txid', 'output') + .map(lambda fulfills: [fulfills['txid'], + fulfills['output']]), + multi=True)) + # wait for rethinkdb to finish creating secondary indexes connection.run( r.db(dbname) diff --git a/tests/backend/mongodb/test_schema.py b/tests/backend/mongodb/test_schema.py index 34b6edf9..71eac7ff 100644 --- a/tests/backend/mongodb/test_schema.py +++ b/tests/backend/mongodb/test_schema.py @@ -21,8 +21,8 @@ def test_init_creates_db_tables_and_indexes(): assert sorted(collection_names) == ['backlog', 'bigchain', 'votes'] indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', - 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', + 'outputs', 'transaction_id'] indexes = conn.conn[dbname]['backlog'].index_information().keys() assert sorted(indexes) == ['_id_', 'assignee__transaction_timestamp', @@ -81,8 +81,8 @@ def test_create_secondary_indexes(): # Bigchain table indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', - 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', + 'outputs', 'transaction_id'] # Backlog table indexes = conn.conn[dbname]['backlog'].index_information().keys() diff --git a/tests/backend/rethinkdb/test_schema.py b/tests/backend/rethinkdb/test_schema.py index 1447e80f..e19dfdc2 100644 --- a/tests/backend/rethinkdb/test_schema.py +++ b/tests/backend/rethinkdb/test_schema.py @@ -85,6 +85,10 @@ def test_create_secondary_indexes(): 'transaction_id')) is True assert conn.run(r.db(dbname).table('bigchain').index_list().contains( 'asset_id')) is True + assert conn.run(r.db(dbname).table('bigchain').index_list().contains( + 'inputs')) is True + assert conn.run(r.db(dbname).table('bigchain').index_list().contains( + 'outputs')) is True # Backlog table assert conn.run(r.db(dbname).table('backlog').index_list().contains(