From aa082277403a6875c179e3c35d98fa71642056ae Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 30 Nov 2016 15:22:36 +0100 Subject: [PATCH 01/21] add some mongodb setup --- bigchaindb/backend/mongodb/__init__.py | 0 bigchaindb/backend/mongodb/connection.py | 46 ++++++++++++++++++++++++ bigchaindb/backend/mongodb/query.py | 42 ++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 bigchaindb/backend/mongodb/__init__.py create mode 100644 bigchaindb/backend/mongodb/connection.py create mode 100644 bigchaindb/backend/mongodb/query.py diff --git a/bigchaindb/backend/mongodb/__init__.py b/bigchaindb/backend/mongodb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py new file mode 100644 index 00000000..1029db16 --- /dev/null +++ b/bigchaindb/backend/mongodb/connection.py @@ -0,0 +1,46 @@ +import time +import logging + +from pymongo import MongoClient +from pymongo.errors import ConnectionError + +import bigchaindb +from bigchaindb.backend.connection import Connection + +logger = logging.getLogger(__name__) + +class MongoDBConnection(Connection): + + def __init__(self, host=None, port=None, db=None, max_tries=3): + """Create a new Connection instance. + + Args: + host (str, optional): the host to connect to. + port (int, optional): the port to connect to. + db (str, optional): the database to use. + max_tries (int, optional): how many tries before giving up. + """ + + self.host = host or bigchaindb.config['database']['host'] + self.port = port or bigchaindb.config['database']['port'] + self.db = db or bigchaindb.config['database']['name'] + self.max_tries = max_tries + self.conn = None + + @property + def db(self): + if self.conn is None: + self._connect() + + else: + return self.conn[self.db] + + def _connect(self): + for i in range(self.max_tries): + try: + self.conn = MongoClient(self.host, self.port) + except ConnectionError as exc: + if i + 1 == self.max_tries: + raise + else: + time.sleep(2**i) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py new file mode 100644 index 00000000..89f9f1e0 --- /dev/null +++ b/bigchaindb/backend/mongodb/query.py @@ -0,0 +1,42 @@ +from bigchaindb.backend import query +from bigchaindb.backend.mongodb.connection import MongoDBConnection + + +@query.write_transaction.register(MongoDBConnection) +def write_transaction(conn, signed_transaction): + return conn.db['backlog'].insert_one(signed_transaction) + + +@query.write_vote.register(MongoDBConnection) +def write_vote(conn, vote): + return conn.db['votes'].insert_one(vote) + + +@query.write_block.register(MongoDBConnection) +def write_block(conn, block): + return conn.db['bigchain'].insert_one(block.to_dict()) + + +@query.genesis_block_exists.register(MongoDBConnection) +def genesis_block_exists(conn): + return conn.db['bigchain'].count() > 0 + + +@query.search_block_election_on_index.register(MongoDBConnection) +def search_block_election_on_index(conn, value, index): + return conn.db['bigchain']\ + .find({index: value}, projection={'votes', 'id', 'block.voters'}) + + +@query.get_votes_on_block.register(MongoDBConnection) +def get_votes_on_block(conn, block_id): + return conn.db['votes'].find({'vote.voting_for_block': block_id}) + + +@query.transaction_exists.register(MongoDBConnection) +def transaction_exists(conn, transaction_id): + if conn.db['bigchain']\ + .find_one({'block.transactions.id': transaction_id}): + return True + else: + return False From eb18d576e9a38c10146386ad38f02ac0d01d2789 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 30 Nov 2016 16:32:50 +0100 Subject: [PATCH 02/21] more mongocalls --- bigchaindb/backend/mongodb/query.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 89f9f1e0..bc9b12db 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -40,3 +40,29 @@ def transaction_exists(conn, transaction_id): return True else: return False + + +@query.get_transaction_from_block.register(MongoDBConnection) +def get_transaction_from_block(conn, block_id, tx_id): + # this is definitely wrong, but it's something like this + return conn.db['bigchain'].find_one({'id': block_id, + 'block.transactions.id': tx_id}) + + +@query.get_tx_by_metadata_id.register(MongoDBConnection) +def get_tx_by_metadata_id(conn, metadata_id): + return conn.db['bigchain']\ + .find({'block.transactions.transaction.metadata.id': metadata_id}) + + +@query.get_txs_by_asset_id.register(MongoDBConnection) +def get_txs_by_asset_id(conn, asset_id): + return conn.db['bigchain']\ + .find({'block.transaction.transaction.asset.id': asset_id}) + + +@query.get_tx_by_fulfillment.register(MongoDBConnection) +def get_tx_by_fulfillment(conn, txid, cid): + return conn.db['bigchain']\ + .find({'block.transactions.transaction.fulfillments.txid': txid, + 'block.transactions.transaction.fulfillments.cid': cid}) From 4877f5b03ae800be956b5a16e6d906d098fcecb5 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 1 Dec 2016 16:24:07 +0100 Subject: [PATCH 03/21] more mongocalls --- bigchaindb/backend/mongodb/query.py | 56 +++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index bc9b12db..4e1cc40d 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -1,3 +1,7 @@ +from time import time + +from pymongo import ReturnDocument + from bigchaindb.backend import query from bigchaindb.backend.mongodb.connection import MongoDBConnection @@ -7,6 +11,44 @@ def write_transaction(conn, signed_transaction): return conn.db['backlog'].insert_one(signed_transaction) +@query.update_transaction.register(MongoDBConnection) +def update_transaction(conn, transaction_id, doc): + return conn.db['backlog']\ + .find_one_and_update({'id': transaction_id}, + doc, + return_document=ReturnDocument.AFTER) + + +@query.delete_transaction.register(MongoDBConnection) +def delete_transaction(conn, *transaction_id): + return conn.db['backlog'].delete_many({'id': {'$in': transaction_id}}) + + +@query.get_stale_transactions.register(MongoDBConnection) +def get_stale_transactions(conn, reassign_delay): + return conn.db['backlog']\ + .find({'assignment_timestamp': {'$lt': time() - reassign_delay}}) + + +@query.get_transaction_from_block.register(MongoDBConnection) +def get_transaction_from_block(conn, block_id, tx_id): + # this is definitely wrong, but it's something like this + return conn.db['bigchain'].find_one({'id': block_id, + 'block.transactions.id': tx_id}) + + +@query.get_transaction_from_backlog(MongoDBConnection) +def get_transaction_from_backlog(conn, transaction_id): + return conn.db['backlog'].find_one({'id': transaction_id}) + + +@query.get_blocks_status_from_transaction(MongoDBConnection) +def get_blocks_status_from_transaction(conn, transaction_id): + return conn.db['bigchain']\ + .find({'block.transactions.id': transaction_id}, + projection=['id', 'block.voters']) + + @query.write_vote.register(MongoDBConnection) def write_vote(conn, vote): return conn.db['votes'].insert_one(vote) @@ -42,13 +84,6 @@ def transaction_exists(conn, transaction_id): return False -@query.get_transaction_from_block.register(MongoDBConnection) -def get_transaction_from_block(conn, block_id, tx_id): - # this is definitely wrong, but it's something like this - return conn.db['bigchain'].find_one({'id': block_id, - 'block.transactions.id': tx_id}) - - @query.get_tx_by_metadata_id.register(MongoDBConnection) def get_tx_by_metadata_id(conn, metadata_id): return conn.db['bigchain']\ @@ -66,3 +101,10 @@ def get_tx_by_fulfillment(conn, txid, cid): return conn.db['bigchain']\ .find({'block.transactions.transaction.fulfillments.txid': txid, 'block.transactions.transaction.fulfillments.cid': cid}) + + +@query.get_owned_ids.register(MongoDBConnection) +def get_owned_ids(conn, owner): + return conn.db['bigchain']\ + .find({'block.transactions.transaction.conditions.owners_after': + owner}) From 2152c245ff4f0072adbe87af541992fe20c561a1 Mon Sep 17 00:00:00 2001 From: ryan Date: Fri, 2 Dec 2016 17:33:37 +0100 Subject: [PATCH 04/21] more mongocalls --- bigchaindb/backend/mongodb/query.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 4e1cc40d..5c64a467 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -37,18 +37,32 @@ def get_transaction_from_block(conn, block_id, tx_id): 'block.transactions.id': tx_id}) -@query.get_transaction_from_backlog(MongoDBConnection) +@query.get_transaction_from_backlog.register(MongoDBConnection) def get_transaction_from_backlog(conn, transaction_id): return conn.db['backlog'].find_one({'id': transaction_id}) -@query.get_blocks_status_from_transaction(MongoDBConnection) +@query.get_blocks_status_from_transaction.register(MongoDBConnection) def get_blocks_status_from_transaction(conn, transaction_id): return conn.db['bigchain']\ .find({'block.transactions.id': transaction_id}, projection=['id', 'block.voters']) +@query.get_txids_by_asset_id.register(MongoDBConnection) +def get_txids_by_asset_id(conn, asset_id): + return conn.db['bigchain']\ + .find({'block.transactions.asset.id': asset_id}, + projection=['id']) + + +@query.get_asset_by_id.register(MongoDBConnection) +def get_asset_by_id(conn, asset_id): + return conn.db['bigchain']\ + .find_one({'block.transactions.asset.id': asset_id}, + projection=['block.transactions.asset']) + + @query.write_vote.register(MongoDBConnection) def write_vote(conn, vote): return conn.db['votes'].insert_one(vote) From ca94574150c34e900eda592867edcd47a873bac5 Mon Sep 17 00:00:00 2001 From: ryan Date: Tue, 6 Dec 2016 16:59:00 +0100 Subject: [PATCH 05/21] more queries --- bigchaindb/backend/mongodb/query.py | 143 ++++++++++++++++++---------- 1 file changed, 91 insertions(+), 52 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 5c64a467..dc2d3b37 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -3,6 +3,7 @@ from time import time from pymongo import ReturnDocument from bigchaindb.backend import query +from bigchaindb.common.exceptions import CyclicBlockchainError from bigchaindb.backend.mongodb.connection import MongoDBConnection @@ -59,62 +60,18 @@ def get_txids_by_asset_id(conn, asset_id): @query.get_asset_by_id.register(MongoDBConnection) def get_asset_by_id(conn, asset_id): return conn.db['bigchain']\ - .find_one({'block.transactions.asset.id': asset_id}, + .find_one({'block.transactions.asset.id': asset_id, + 'block.transactions.asset.operation': 'CREATE'}, projection=['block.transactions.asset']) -@query.write_vote.register(MongoDBConnection) -def write_vote(conn, vote): - return conn.db['votes'].insert_one(vote) - - -@query.write_block.register(MongoDBConnection) -def write_block(conn, block): - return conn.db['bigchain'].insert_one(block.to_dict()) - - -@query.genesis_block_exists.register(MongoDBConnection) -def genesis_block_exists(conn): - return conn.db['bigchain'].count() > 0 - - -@query.search_block_election_on_index.register(MongoDBConnection) -def search_block_election_on_index(conn, value, index): +@query.get_spent.register(MongoDBConnection) +def get_spent(conn, transaction_id, condition_id): return conn.db['bigchain']\ - .find({index: value}, projection={'votes', 'id', 'block.voters'}) - - -@query.get_votes_on_block.register(MongoDBConnection) -def get_votes_on_block(conn, block_id): - return conn.db['votes'].find({'vote.voting_for_block': block_id}) - - -@query.transaction_exists.register(MongoDBConnection) -def transaction_exists(conn, transaction_id): - if conn.db['bigchain']\ - .find_one({'block.transactions.id': transaction_id}): - return True - else: - return False - - -@query.get_tx_by_metadata_id.register(MongoDBConnection) -def get_tx_by_metadata_id(conn, metadata_id): - return conn.db['bigchain']\ - .find({'block.transactions.transaction.metadata.id': metadata_id}) - - -@query.get_txs_by_asset_id.register(MongoDBConnection) -def get_txs_by_asset_id(conn, asset_id): - return conn.db['bigchain']\ - .find({'block.transaction.transaction.asset.id': asset_id}) - - -@query.get_tx_by_fulfillment.register(MongoDBConnection) -def get_tx_by_fulfillment(conn, txid, cid): - return conn.db['bigchain']\ - .find({'block.transactions.transaction.fulfillments.txid': txid, - 'block.transactions.transaction.fulfillments.cid': cid}) + .find_one({'block.transactions.fulfillments.input.txid': + transaction_id, + 'block.transactions.fulfillments.input.cid': + condition_id}) @query.get_owned_ids.register(MongoDBConnection) @@ -122,3 +79,85 @@ def get_owned_ids(conn, owner): return conn.db['bigchain']\ .find({'block.transactions.transaction.conditions.owners_after': owner}) + + +@query.get_votes_by_block_id.register(MongoDBConnection) +def get_votes_by_block_id(conn, block_id): + return conn.db['votes']\ + .find({'vote.voting_for_block': block_id}) + + +@query.get_votes_by_block_id_and_voter.register(MongoDBConnection) +def get_votes_block_id_and_voter(conn, block_id, node_pubkey): + return conn.db['votes']\ + .find({'vote.voting_for_block': block_id, + 'node_pubkey': node_pubkey}) + + +@query.write_block.register(MongoDBConnection) +def write_block(conn, block): + return conn.db['bigchain'].insert_one(block.to_dict()) + + +@query.get_block.register(MongoDBConnection) +def get_block(conn, block_id): + return conn.db['bigchain'].find_one({'id': block_id}) + + +@query.has_transaction.register(MongoDBConnection) +def has_transaction(conn, transaction_id): + return bool(conn.db['bigchain'] + .find_one({'block.transactions.id': transaction_id})) + + +@query.count_blocks.register(MongoDBConnection) +def count_blocks(conn): + return conn.db['bigchain'].count() + + +@query.count_backlog(MongoDBConnection) +def count_backlog(conn): + return conn.db['backlog'].count() + + +@query.write_vote.register(MongoDBConnection) +def write_vote(conn, vote): + return conn.db['votes'].insert_one(vote) + + +@query.get_genesis_block.register(MongoDBConnection) +def get_genesis_block(conn): + return conn.db['bigchain'].find_one({'block.transactions.0.operation' == + 'GENESIS'}) + + +@query.get_last_voted_block.register(MongoDBConnection) +def get_last_voted_block(conn, node_pubkey): + last_voted = conn.db['votes']\ + .find({'node_pubkey': node_pubkey}, + sort=[('vote.timestamp', -1)]) + if not last_voted: + return get_genesis_block(conn) + + mapping = {v['vote']['previous_block']: v['vote']['voting_for_block'] + for v in last_voted} + + last_block_id = list(mapping.values())[0] + + explored = set() + + while True: + try: + if last_block_id in explored: + raise CyclicBlockchainError() + explored.add(last_block_id) + last_block_id = mapping[last_block_id] + except KeyError: + break + + return get_block(conn, last_block_id) + + +@query.get_unvoted_blocks.register(MongoDBConnection) +def get_unvoted_blocks(conn, node_pubkey): + pass From f3cc167edb35f6e39e1b698235c953a21b92be73 Mon Sep 17 00:00:00 2001 From: ryan Date: Tue, 6 Dec 2016 17:24:20 +0100 Subject: [PATCH 06/21] refactor --- bigchaindb/backend/mongodb/query.py | 52 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index dc2d3b37..eacbcea7 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -1,18 +1,24 @@ +"""Query implementation for MongoDB""" + from time import time from pymongo import ReturnDocument -from bigchaindb.backend import query +from bigchaindb import backend from bigchaindb.common.exceptions import CyclicBlockchainError +from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.mongodb.connection import MongoDBConnection -@query.write_transaction.register(MongoDBConnection) +register_query = module_dispatch_registrar(backend.query) + + +@register_query(MongoDBConnection) def write_transaction(conn, signed_transaction): return conn.db['backlog'].insert_one(signed_transaction) -@query.update_transaction.register(MongoDBConnection) +@register_query(MongoDBConnection) def update_transaction(conn, transaction_id, doc): return conn.db['backlog']\ .find_one_and_update({'id': transaction_id}, @@ -20,44 +26,44 @@ def update_transaction(conn, transaction_id, doc): return_document=ReturnDocument.AFTER) -@query.delete_transaction.register(MongoDBConnection) +@register_query(MongoDBConnection) def delete_transaction(conn, *transaction_id): return conn.db['backlog'].delete_many({'id': {'$in': transaction_id}}) -@query.get_stale_transactions.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_stale_transactions(conn, reassign_delay): return conn.db['backlog']\ .find({'assignment_timestamp': {'$lt': time() - reassign_delay}}) -@query.get_transaction_from_block.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_transaction_from_block(conn, block_id, tx_id): # this is definitely wrong, but it's something like this return conn.db['bigchain'].find_one({'id': block_id, 'block.transactions.id': tx_id}) -@query.get_transaction_from_backlog.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_transaction_from_backlog(conn, transaction_id): return conn.db['backlog'].find_one({'id': transaction_id}) -@query.get_blocks_status_from_transaction.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_blocks_status_from_transaction(conn, transaction_id): return conn.db['bigchain']\ .find({'block.transactions.id': transaction_id}, projection=['id', 'block.voters']) -@query.get_txids_by_asset_id.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_txids_by_asset_id(conn, asset_id): return conn.db['bigchain']\ .find({'block.transactions.asset.id': asset_id}, projection=['id']) -@query.get_asset_by_id.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_asset_by_id(conn, asset_id): return conn.db['bigchain']\ .find_one({'block.transactions.asset.id': asset_id, @@ -65,7 +71,7 @@ def get_asset_by_id(conn, asset_id): projection=['block.transactions.asset']) -@query.get_spent.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_spent(conn, transaction_id, condition_id): return conn.db['bigchain']\ .find_one({'block.transactions.fulfillments.input.txid': @@ -74,64 +80,64 @@ def get_spent(conn, transaction_id, condition_id): condition_id}) -@query.get_owned_ids.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_owned_ids(conn, owner): return conn.db['bigchain']\ .find({'block.transactions.transaction.conditions.owners_after': owner}) -@query.get_votes_by_block_id.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_votes_by_block_id(conn, block_id): return conn.db['votes']\ .find({'vote.voting_for_block': block_id}) -@query.get_votes_by_block_id_and_voter.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_votes_block_id_and_voter(conn, block_id, node_pubkey): return conn.db['votes']\ .find({'vote.voting_for_block': block_id, 'node_pubkey': node_pubkey}) -@query.write_block.register(MongoDBConnection) +@register_query(MongoDBConnection) def write_block(conn, block): return conn.db['bigchain'].insert_one(block.to_dict()) -@query.get_block.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_block(conn, block_id): return conn.db['bigchain'].find_one({'id': block_id}) -@query.has_transaction.register(MongoDBConnection) +@register_query(MongoDBConnection) def has_transaction(conn, transaction_id): return bool(conn.db['bigchain'] .find_one({'block.transactions.id': transaction_id})) -@query.count_blocks.register(MongoDBConnection) +@register_query(MongoDBConnection) def count_blocks(conn): return conn.db['bigchain'].count() -@query.count_backlog(MongoDBConnection) +@register_query(MongoDBConnection) def count_backlog(conn): return conn.db['backlog'].count() -@query.write_vote.register(MongoDBConnection) +@register_query(MongoDBConnection) def write_vote(conn, vote): return conn.db['votes'].insert_one(vote) -@query.get_genesis_block.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_genesis_block(conn): return conn.db['bigchain'].find_one({'block.transactions.0.operation' == 'GENESIS'}) -@query.get_last_voted_block.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_last_voted_block(conn, node_pubkey): last_voted = conn.db['votes']\ .find({'node_pubkey': node_pubkey}, @@ -158,6 +164,6 @@ def get_last_voted_block(conn, node_pubkey): return get_block(conn, last_block_id) -@query.get_unvoted_blocks.register(MongoDBConnection) +@register_query(MongoDBConnection) def get_unvoted_blocks(conn, node_pubkey): pass From f9285fdb293de25889d20d48af0d1b41d7d20b91 Mon Sep 17 00:00:00 2001 From: ryan Date: Tue, 6 Dec 2016 18:18:03 +0100 Subject: [PATCH 07/21] add schema --- bigchaindb/backend/mongodb/schema.py | 104 +++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 bigchaindb/backend/mongodb/schema.py diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py new file mode 100644 index 00000000..7cfcc18d --- /dev/null +++ b/bigchaindb/backend/mongodb/schema.py @@ -0,0 +1,104 @@ +"""Utils to initialize and drop the database.""" + +import logging + +from pymongo import ASCENDING, DESCENDING + +from bigchaindb import backend +from bigchaindb.common import exceptions +from bigchaindb.backend.utils import module_dispatch_registrar +from bigchaindb.backend.mongodb.connection import MongoDBConnection + + +logger = logging.getLogger(__name__) +register_schema = module_dispatch_registrar(backend.schema) + + +@register_schema(MongoDBConnection) +def create_database(conn, dbname): + if dbname in conn.database_names(): + raise exceptions.DatabaseAlreadyExists('Database `{}` already exists' + .format(dbname)) + + logger.info('Create database `%s`.', dbname) + # TODO: read and write concerns can be declared here + conn.get_database(dbname) + + +@register_schema(MongoDBConnection) +def create_table(conn, dbname): + for table_name in ['bigchain', 'backlog', 'votes']: + logger.info('Create `%s` table.', table_name) + # create the table + # TODO: read and write concerns can be declared here + conn[dbname].create_collection(table_name) + + +@register_schema(MongoDBConnection) +def create_indexes(conn, dbname): + create_bigchain_secondary_index(conn, dbname) + create_backlog_secondary_index(conn, dbname) + create_votes_secondary_index(conn, dbname) + + +@register_schema(MongoDBConnection) +def drop(conn, dbname): + conn.drop_database(dbname) + + +def create_bigchain_secondary_index(conn, dbname): + logger.info('Create `bigchain` secondary index.') + + # to select blocks by id + conn[dbname]['bigchain'].create_index('id', name='block_id') + + # to order blocks by timestamp + conn[dbname]['bigchain'].create_index('block.timestamp', ASCENDING, + name='block_timestamp') + + # to query the bigchain for a transaction id, this field is unique + conn[dbname]['bigchain'].create_index('block.transactions.id', + name='transaction_id', unique=True) + + # secondary index for payload data by UUID, this field is unique + conn[dbname]['bigchain']\ + .create_index('block.transactions.transaction.metadata.id', + name='metadata_id', unique=True) + + # secondary index for asset uuid, this field is unique + conn[dbname]['bigchain']\ + .create_index('block.transactions.transaction.asset.id', + name='asset_id', unique=True) + + # compound index on fulfillment and transactions id + conn[dbname]['bigchain']\ + .create_index(['block.transactions.transaction.fulfillments.txid', + 'block.transactions.transaction.fulfillments.cid'], + name='tx_and_fulfillment') + + +def create_backlog_secondary_index(conn, dbname): + logger.info('Create `backlog` secondary index.') + + # to order transactions by timestamp + conn[dbname]['backlog'].create_index('transaction.timestamp', ASCENDING, + name='transaction_timestamp') + + # compound index to read transactions from the backlog per assignee + conn[dbname]['backlog']\ + .create_index(['assignee', + ('assignment_timestamp', DESCENDING)], + name='assignee__transaction_timestamp') + + +def create_votes_secondary_index(conn, dbname): + logger.info('Create `votes` secondary index.') + + # index on block id to quickly poll + conn[dbname]['votes'].create_index('vote.voting_for_block', + name='voting_for') + + # is the first index redundant then? + # compound index to order votes by block id and node + conn[dbname]['votes'].create_index(['vote.voting_for_block', + 'node_pubkey'], name='block_and_voter') From 9bff8b40b5b168c51eca3a3e5516dfb9fed8d13b Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:31:04 +0100 Subject: [PATCH 08/21] Add mongodb container service for dev/tests --- docker-compose.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cdd88078..0d20f868 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,11 @@ version: '2' services: + mdb: + image: mongo + ports: + - "27017" + rdb: image: rethinkdb ports: @@ -30,8 +35,30 @@ services: - ./tox.ini:/usr/src/app/tox.ini - ./Makefile:/usr/src/app/Makefile environment: + BIGCHAINDB_DATABASE_BACKEND: rethinkdb BIGCHAINDB_DATABASE_HOST: rdb BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 ports: - "9984" command: bigchaindb start + + bdb-mdb: + build: + context: . + dockerfile: Dockerfile-dev + volumes: + - ./bigchaindb:/usr/src/app/bigchaindb + - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini + environment: + BIGCHAINDB_DATABASE_BACKEND: mongodb + BIGCHAINDB_DATABASE_HOST: mdb + BIGCHAINDB_DATABASE_PORT: 27017 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + ports: + - "9984" + command: bigchaindb start From 11d0f4bff5d2fe5c6cb950c8b1e2cdf7fa841e58 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:39:54 +0100 Subject: [PATCH 09/21] Import modules of registered (dispatch) functions --- bigchaindb/backend/mongodb/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigchaindb/backend/mongodb/__init__.py b/bigchaindb/backend/mongodb/__init__.py index e69de29b..b49bf6e6 100644 --- a/bigchaindb/backend/mongodb/__init__.py +++ b/bigchaindb/backend/mongodb/__init__.py @@ -0,0 +1,2 @@ +# Register the single dispatched modules on import. +from bigchaindb.backend.mongodb import schema, query # noqa From d3d42747b7f8724bdeef38eb2488355071eb5525 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:42:49 +0100 Subject: [PATCH 10/21] Add mongodb backend --- bigchaindb/backend/connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 88c0eae4..1d7dda03 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -5,7 +5,8 @@ from bigchaindb.common.exceptions import ConfigurationError BACKENDS = { - 'rethinkdb': 'bigchaindb.backend.rethinkdb.connection.RethinkDBConnection' + 'rethinkdb': 'bigchaindb.backend.rethinkdb.connection.RethinkDBConnection', + 'mongodb': 'bigchaindb.backend.mongodb.connection.MongoDBConnection', } From 1f996c174659328473ab437a0fbcbce1f6df2a9a Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:44:08 +0100 Subject: [PATCH 11/21] Import ConnectionFailure exception --- bigchaindb/backend/mongodb/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 1029db16..a93fb29f 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -2,7 +2,7 @@ import time import logging from pymongo import MongoClient -from pymongo.errors import ConnectionError +from pymongo.errors import ConnectionFailure import bigchaindb from bigchaindb.backend.connection import Connection @@ -39,7 +39,7 @@ class MongoDBConnection(Connection): for i in range(self.max_tries): try: self.conn = MongoClient(self.host, self.port) - except ConnectionError as exc: + except ConnectionFailure as exc: if i + 1 == self.max_tries: raise else: From 14d55643498da38cbcf34966ca9b071fe030932e Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:45:41 +0100 Subject: [PATCH 12/21] Fix some problem with class attribute and property --- bigchaindb/backend/mongodb/connection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index a93fb29f..3a967d48 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -11,19 +11,19 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, host=None, port=None, db=None, max_tries=3): + def __init__(self, host=None, port=None, dbname=None, max_tries=3): """Create a new Connection instance. Args: host (str, optional): the host to connect to. port (int, optional): the port to connect to. - db (str, optional): the database to use. + dbname (str, optional): the database to use. max_tries (int, optional): how many tries before giving up. """ self.host = host or bigchaindb.config['database']['host'] self.port = port or bigchaindb.config['database']['port'] - self.db = db or bigchaindb.config['database']['name'] + self.dbname = dbname or bigchaindb.config['database']['name'] self.max_tries = max_tries self.conn = None @@ -33,7 +33,7 @@ class MongoDBConnection(Connection): self._connect() else: - return self.conn[self.db] + return self.conn[self.dbname] def _connect(self): for i in range(self.max_tries): From bd886553d4424bba5467a063e97c86445a198707 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:46:27 +0100 Subject: [PATCH 13/21] Add missing blank line --- bigchaindb/backend/mongodb/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 3a967d48..3b6289f4 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -9,6 +9,7 @@ from bigchaindb.backend.connection import Connection logger = logging.getLogger(__name__) + class MongoDBConnection(Connection): def __init__(self, host=None, port=None, dbname=None, max_tries=3): From a902f895faf795c50484fd842559c14e4bed5db6 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:46:56 +0100 Subject: [PATCH 14/21] Update function names to match the generic ones --- bigchaindb/backend/mongodb/query.py | 2 +- bigchaindb/backend/mongodb/schema.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index eacbcea7..369cb39d 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -94,7 +94,7 @@ def get_votes_by_block_id(conn, block_id): @register_query(MongoDBConnection) -def get_votes_block_id_and_voter(conn, block_id, node_pubkey): +def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): return conn.db['votes']\ .find({'vote.voting_for_block': block_id, 'node_pubkey': node_pubkey}) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 7cfcc18d..1a5d2aec 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -26,7 +26,7 @@ def create_database(conn, dbname): @register_schema(MongoDBConnection) -def create_table(conn, dbname): +def create_tables(conn, dbname): for table_name in ['bigchain', 'backlog', 'votes']: logger.info('Create `%s` table.', table_name) # create the table @@ -42,7 +42,7 @@ def create_indexes(conn, dbname): @register_schema(MongoDBConnection) -def drop(conn, dbname): +def drop_database(conn, dbname): conn.drop_database(dbname) From b87baaaa1ded4c1ab0c75c38a7610e8f62363d0c Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:47:50 +0100 Subject: [PATCH 15/21] Test backend.schema.init_database --- tests/backend/test_generics.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index b11d534a..acdd537e 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -1,3 +1,6 @@ +from importlib import import_module +from unittest.mock import patch + from pytest import mark, raises @@ -43,3 +46,26 @@ def test_query(query_func_name, args_qty): query_func = getattr(query, query_func_name) with raises(NotImplementedError): query_func(None, *range(args_qty)) + + +@mark.parametrize('db,conn_cls', ( + ('mongodb', 'MongoDBConnection'), + ('rethinkdb', 'RethinkDBConnection'), +)) +@patch('bigchaindb.backend.schema.create_indexes', + autospec=True, return_value=None) +@patch('bigchaindb.backend.schema.create_tables', + autospec=True, return_value=None) +@patch('bigchaindb.backend.schema.create_database', + autospec=True, return_value=None) +def test_init_database(mock_create_database, mock_create_tables, + mock_create_indexes, db, conn_cls): + from bigchaindb.backend.schema import init_database + conn = getattr( + import_module('bigchaindb.backend.{}.connection'.format(db)), + conn_cls, + )('host', 'port', 'dbname') + init_database(connection=conn, dbname='mickeymouse') + mock_create_database.assert_called_once_with(conn, 'mickeymouse') + mock_create_tables.assert_called_once_with(conn, 'mickeymouse') + mock_create_indexes.assert_called_once_with(conn, 'mickeymouse') From 5d0c7fabf960406b387f8fdaae10e6a71a7ca6cf Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 13 Dec 2016 11:48:34 +0100 Subject: [PATCH 16/21] Add a bit of logging --- bigchaindb/backend/connection.py | 4 ++++ bigchaindb/backend/schema.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 1d7dda03..4ba611d5 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -1,4 +1,5 @@ from importlib import import_module +import logging import bigchaindb from bigchaindb.common.exceptions import ConfigurationError @@ -9,6 +10,8 @@ BACKENDS = { 'mongodb': 'bigchaindb.backend.mongodb.connection.MongoDBConnection', } +logger = logging.getLogger(__name__) + def connect(backend=None, host=None, port=None, name=None): """Create a new connection to the database backend. @@ -45,6 +48,7 @@ def connect(backend=None, host=None, port=None, name=None): except (ImportError, AttributeError) as exc: raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc + logger.debug('Connection: {}'.format(Class)) return Class(host, port, dbname) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index b731a55f..086854b0 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -1,10 +1,13 @@ """Database creation and schema-providing interfaces for backends.""" from functools import singledispatch +import logging import bigchaindb from bigchaindb.backend.connection import connect +logger = logging.getLogger(__name__) + @singledispatch def create_database(connection, dbname): From 33480bead2b928da94be07dbbb362670826bce13 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 14 Dec 2016 17:21:30 +0100 Subject: [PATCH 17/21] remove rethinkdb calls --- tests/db/conftest.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/db/conftest.py b/tests/db/conftest.py index 4959e2d2..7ccf02d9 100644 --- a/tests/db/conftest.py +++ b/tests/db/conftest.py @@ -7,12 +7,12 @@ Tasks: """ import pytest -import rethinkdb as r from bigchaindb import Bigchain from bigchaindb.backend import connect, schema from bigchaindb.common import crypto -from bigchaindb.common.exceptions import DatabaseAlreadyExists +from bigchaindb.common.exceptions import (DatabaseAlreadyExists, + DatabaseDoesNotExist) USER2_SK, USER2_PK = crypto.generate_key_pair() @@ -30,23 +30,22 @@ def setup_database(request, node_config): db_name = node_config['database']['name'] conn = connect() - if conn.run(r.db_list().contains(db_name)): - conn.run(r.db_drop(db_name)) - try: - schema.init_database() + schema.init_database(conn) except DatabaseAlreadyExists: print('Database already exists.') + schema.drop_database(conn, db_name) + schema.init_database(conn) print('Finishing init database') def fin(): - print('Deleting `{}` database'.format(db_name)) conn = connect() + print('Deleting `{}` database'.format(db_name)) try: - conn.run(r.db_drop(db_name)) - except r.ReqlOpFailedError as e: - if e.message != 'Database `{}` does not exist.'.format(db_name): + schema.drop_database(conn, db_name) + except DatabaseDoesNotExist as e: + if str(e) != 'Database `{}` does not exist'.format(db_name): raise print('Finished deleting `{}`'.format(db_name)) @@ -60,11 +59,12 @@ def cleanup_tables(request, node_config): def fin(): conn = connect() try: - conn.run(r.db(db_name).table('bigchain').delete()) - conn.run(r.db(db_name).table('backlog').delete()) - conn.run(r.db(db_name).table('votes').delete()) - except r.ReqlOpFailedError as e: - if e.message != 'Database `{}` does not exist.'.format(db_name): + schema.drop_database(conn, db_name) + schema.create_database(conn, db_name) + schema.create_tables(conn, db_name) + schema.create_indexes(conn, db_name) + except DatabaseDoesNotExist as e: + if str(e) != 'Database `{}` does not exist'.format(db_name): raise request.addfinalizer(fin) From 243c739d2b7d6eacd37aa1cdb3457da44a2e24eb Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 15 Dec 2016 15:06:07 +0100 Subject: [PATCH 18/21] config changes --- bigchaindb/backend/connection.py | 2 +- bigchaindb/backend/mongodb/__init__.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 4ba611d5..2d4f9bac 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -6,8 +6,8 @@ from bigchaindb.common.exceptions import ConfigurationError BACKENDS = { - 'rethinkdb': 'bigchaindb.backend.rethinkdb.connection.RethinkDBConnection', 'mongodb': 'bigchaindb.backend.mongodb.connection.MongoDBConnection', + 'rethinkdb': 'bigchaindb.backend.rethinkdb.connection.RethinkDBConnection' } logger = logging.getLogger(__name__) diff --git a/bigchaindb/backend/mongodb/__init__.py b/bigchaindb/backend/mongodb/__init__.py index b49bf6e6..2bc65591 100644 --- a/bigchaindb/backend/mongodb/__init__.py +++ b/bigchaindb/backend/mongodb/__init__.py @@ -1,2 +1,22 @@ +"""MongoDB backend implementation. + +Contains a MongoDB-specific implementation of the +:mod:`~bigchaindb.backend.changefeed`, :mod:`~bigchaindb.backend.query`, and +:mod:`~bigchaindb.backend.schema` interfaces. + +You can specify BigchainDB to use MongoDB as its database backend by either +setting ``database.backend`` to ``'rethinkdb'`` in your configuration file, or +setting the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable to +``'rethinkdb'``. + +If configured to use MongoDB, BigchainDB will automatically return instances +of :class:`~bigchaindb.backend.rethinkdb.MongoDBConnection` for +:func:`~bigchaindb.backend.connection.connect` and dispatch calls of the +generic backend interfaces to the implementations in this module. +""" + # Register the single dispatched modules on import. -from bigchaindb.backend.mongodb import schema, query # noqa +from bigchaindb.backend.mongodb import changefeed, schema, query # noqa + +# MongoDBConnection should always be accessed via +# ``bigchaindb.backend.connect()``. From dcba5421f14cbcabfaa6a3704757d012d993ad4a Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 15 Dec 2016 16:02:26 +0100 Subject: [PATCH 19/21] schema fixes --- bigchaindb/backend/mongodb/__init__.py | 2 +- bigchaindb/backend/mongodb/connection.py | 10 ++++-- bigchaindb/backend/mongodb/schema.py | 43 +++++++++++++----------- tests/conftest.py | 3 ++ 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/bigchaindb/backend/mongodb/__init__.py b/bigchaindb/backend/mongodb/__init__.py index 2bc65591..b43a9c24 100644 --- a/bigchaindb/backend/mongodb/__init__.py +++ b/bigchaindb/backend/mongodb/__init__.py @@ -16,7 +16,7 @@ generic backend interfaces to the implementations in this module. """ # Register the single dispatched modules on import. -from bigchaindb.backend.mongodb import changefeed, schema, query # noqa +from bigchaindb.backend.mongodb import schema, query # noqa no changefeed for now # MongoDBConnection should always be accessed via # ``bigchaindb.backend.connect()``. diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 3b6289f4..f50d8416 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -26,7 +26,13 @@ class MongoDBConnection(Connection): self.port = port or bigchaindb.config['database']['port'] self.dbname = dbname or bigchaindb.config['database']['name'] self.max_tries = max_tries - self.conn = None + self.connection = None + + @property + def conn(self): + if self.connection is None: + self._connect() + return self.connection @property def db(self): @@ -39,7 +45,7 @@ class MongoDBConnection(Connection): def _connect(self): for i in range(self.max_tries): try: - self.conn = MongoClient(self.host, self.port) + self.connection = MongoClient(self.host, self.port) except ConnectionFailure as exc: if i + 1 == self.max_tries: raise diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 1a5d2aec..eadd815f 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -16,13 +16,13 @@ register_schema = module_dispatch_registrar(backend.schema) @register_schema(MongoDBConnection) def create_database(conn, dbname): - if dbname in conn.database_names(): + if dbname in conn.conn.database_names(): raise exceptions.DatabaseAlreadyExists('Database `{}` already exists' .format(dbname)) logger.info('Create database `%s`.', dbname) # TODO: read and write concerns can be declared here - conn.get_database(dbname) + conn.conn.get_database(dbname) @register_schema(MongoDBConnection) @@ -31,7 +31,7 @@ def create_tables(conn, dbname): logger.info('Create `%s` table.', table_name) # create the table # TODO: read and write concerns can be declared here - conn[dbname].create_collection(table_name) + conn.conn[dbname].create_collection(table_name) @register_schema(MongoDBConnection) @@ -43,37 +43,39 @@ def create_indexes(conn, dbname): @register_schema(MongoDBConnection) def drop_database(conn, dbname): - conn.drop_database(dbname) + conn.conn.drop_database(dbname) def create_bigchain_secondary_index(conn, dbname): logger.info('Create `bigchain` secondary index.') # to select blocks by id - conn[dbname]['bigchain'].create_index('id', name='block_id') + conn.conn[dbname]['bigchain'].create_index('id', name='block_id') # to order blocks by timestamp - conn[dbname]['bigchain'].create_index('block.timestamp', ASCENDING, + conn.conn[dbname]['bigchain'].create_index([('block.timestamp', ASCENDING)], name='block_timestamp') # to query the bigchain for a transaction id, this field is unique - conn[dbname]['bigchain'].create_index('block.transactions.id', + conn.conn[dbname]['bigchain'].create_index('block.transactions.id', name='transaction_id', unique=True) # secondary index for payload data by UUID, this field is unique - conn[dbname]['bigchain']\ + conn.conn[dbname]['bigchain']\ .create_index('block.transactions.transaction.metadata.id', name='metadata_id', unique=True) # secondary index for asset uuid, this field is unique - conn[dbname]['bigchain']\ + conn.conn[dbname]['bigchain']\ .create_index('block.transactions.transaction.asset.id', name='asset_id', unique=True) # compound index on fulfillment and transactions id - conn[dbname]['bigchain']\ - .create_index(['block.transactions.transaction.fulfillments.txid', - 'block.transactions.transaction.fulfillments.cid'], + conn.conn[dbname]['bigchain']\ + .create_index([('block.transactions.transaction.fulfillments.txid', + ASCENDING), + ('block.transactions.transaction.fulfillments.cid', + ASCENDING)], name='tx_and_fulfillment') @@ -81,12 +83,13 @@ def create_backlog_secondary_index(conn, dbname): logger.info('Create `backlog` secondary index.') # to order transactions by timestamp - conn[dbname]['backlog'].create_index('transaction.timestamp', ASCENDING, - name='transaction_timestamp') + conn.conn[dbname]['backlog']\ + .create_index([('transaction.timestamp', ASCENDING)], + name='transaction_timestamp') # compound index to read transactions from the backlog per assignee - conn[dbname]['backlog']\ - .create_index(['assignee', + conn.conn[dbname]['backlog']\ + .create_index([('assignee', ASCENDING), ('assignment_timestamp', DESCENDING)], name='assignee__transaction_timestamp') @@ -95,10 +98,12 @@ def create_votes_secondary_index(conn, dbname): logger.info('Create `votes` secondary index.') # index on block id to quickly poll - conn[dbname]['votes'].create_index('vote.voting_for_block', + conn.conn[dbname]['votes'].create_index('vote.voting_for_block', name='voting_for') # is the first index redundant then? # compound index to order votes by block id and node - conn[dbname]['votes'].create_index(['vote.voting_for_block', - 'node_pubkey'], name='block_and_voter') + conn.conn[dbname]['votes']\ + .create_index([('vote.voting_for_block', ASCENDING), + ('node_pubkey', ASCENDING)], + name='block_and_voter') diff --git a/tests/conftest.py b/tests/conftest.py index 1830ab3c..bddfa47b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,6 +64,9 @@ def restore_config(request, node_config): def node_config(request): config = copy.deepcopy(CONFIG) config['database']['backend'] = request.config.getoption('--database-backend') + if config['database']['backend'] == 'mongodb': + # not a great way to do this + config['database']['port'] = 27017 return config From 72ca5e9ddea11fa43e129249775210f2cb0c4fb5 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 16 Dec 2016 13:06:16 +0100 Subject: [PATCH 20/21] fixed some pep8 issues --- bigchaindb/backend/mongodb/schema.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index eadd815f..55fd955d 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -53,12 +53,14 @@ def create_bigchain_secondary_index(conn, dbname): conn.conn[dbname]['bigchain'].create_index('id', name='block_id') # to order blocks by timestamp - conn.conn[dbname]['bigchain'].create_index([('block.timestamp', ASCENDING)], - name='block_timestamp') + conn.conn[dbname]['bigchain'].create_index([('block.timestamp', + ASCENDING)], + name='block_timestamp') # to query the bigchain for a transaction id, this field is unique conn.conn[dbname]['bigchain'].create_index('block.transactions.id', - name='transaction_id', unique=True) + name='transaction_id', + unique=True) # secondary index for payload data by UUID, this field is unique conn.conn[dbname]['bigchain']\ @@ -83,9 +85,9 @@ def create_backlog_secondary_index(conn, dbname): logger.info('Create `backlog` secondary index.') # to order transactions by timestamp - conn.conn[dbname]['backlog']\ - .create_index([('transaction.timestamp', ASCENDING)], - name='transaction_timestamp') + conn.conn[dbname]['backlog'].create_index([('transaction.timestamp', + ASCENDING)], + name='transaction_timestamp') # compound index to read transactions from the backlog per assignee conn.conn[dbname]['backlog']\ @@ -99,11 +101,12 @@ def create_votes_secondary_index(conn, dbname): # index on block id to quickly poll conn.conn[dbname]['votes'].create_index('vote.voting_for_block', - name='voting_for') + name='voting_for') # is the first index redundant then? # compound index to order votes by block id and node - conn.conn[dbname]['votes']\ - .create_index([('vote.voting_for_block', ASCENDING), - ('node_pubkey', ASCENDING)], - name='block_and_voter') + conn.conn[dbname]['votes'].create_index([('vote.voting_for_block', + ASCENDING), + ('node_pubkey', + ASCENDING)], + name='block_and_voter') From ab56d87bcb8e3684522ab8c8c284e5d1248f1d2e Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 16 Dec 2016 13:36:59 +0100 Subject: [PATCH 21/21] codecov ignores mongodb folder for now --- codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codecov.yml b/codecov.yml index b6f22af9..877db0d3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -32,6 +32,7 @@ coverage: - "benchmarking-tests/*" - "speed-tests/*" - "ntools/*" + - "bigchaindb/backend/mongodb/*" comment: # @stevepeak (from codecov.io) suggested we change 'suggestions' to 'uncovered'