diff --git a/CHANGELOG.md b/CHANGELOG.md index 3db903e3..4bfbd8dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,20 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [0.10.2] - 2017-05-16 +Tag name: v0.10.2 + +## Added +* Add Cross Origin Resource Sharing (CORS) support for the HTTP API. + [Commit 6cb7596](https://github.com/bigchaindb/bigchaindb/commit/6cb75960b05403c77bdae0fd327612482589efcb) + +## Fixed +* Fixed `streams_v1` API link in response to `GET /api/v1`. + [Pull Request #1466](https://github.com/bigchaindb/bigchaindb/pull/1466) +* Fixed mismatch between docs and implementation for `GET /blocks?status=` + endpoint. The `status` query parameter is now case insensitive. + [Pull Request #1464](https://github.com/bigchaindb/bigchaindb/pull/1464) + ## [0.10.1] - 2017-04-19 Tag name: v0.10.1 diff --git a/Release_Process.md b/Release_Process.md index e4a988a1..be4c448a 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -10,6 +10,7 @@ that [major version 0.x does not export a stable API](http://semver.org/#spec-it A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server. 1. Update the `CHANGELOG.md` file in master +1. In `k8s/bigchaindb/bigchaindb-dep.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.) Commit that change to master 1. Create and checkout a new branch for the minor release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` (*not* 0.9.0, this new branch will be for e.g. 0.9.0, 0.9.1, 0.9.2, etc. each of which will be identified by a tagged commit) 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) 1. Commit that change, and push the new branch to GitHub @@ -26,8 +27,11 @@ A patch release is similar to a minor release, but piggybacks on an existing min 1. Check out the minor release branch, e.g. `0.9` 1. Apply the changes you want, e.g. using `git cherry-pick`. 1. Update the `CHANGELOG.md` file -1. Increment the patch version in `bigchaindb/version.py`, e.g. "0.9.1" -1. Commit that change, and push the updated branch to GitHub +1. Increment the patch version in `bigchaindb/version.py`, e.g. `0.9.1` +1. Commit that change +1. In `k8s/bigchaindb/bigchaindb-dep.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.9.0` and change the version number to the new version number, e.g. `0.9.1`. (This is the Docker image that Kubernetes should pull from Docker Hub.) +1. Commit that change +1. Push the updated minor release branch to GitHub 1. Follow steps outlined in [Common Steps](#common-steps) 1. Cherry-pick the `CHANGELOG.md` update commit (made above) to the `master` branch diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 74b9c35a..eaef721d 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -7,7 +7,7 @@ from pymongo import ReturnDocument from bigchaindb import backend from bigchaindb.common.exceptions import CyclicBlockchainError from bigchaindb.common.transaction import Transaction -from bigchaindb.backend.exceptions import DuplicateKeyError +from bigchaindb.backend.exceptions import DuplicateKeyError, OperationError from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.mongodb.connection import MongoDBConnection @@ -127,6 +127,7 @@ def get_txids_filtered(conn, asset_id, operation=None): return (elem['block']['transactions']['id'] for elem in cursor) +# TODO: This doesn't seem to be used anywhere @register_query(MongoDBConnection) def get_asset_by_id(conn, asset_id): cursor = conn.run( @@ -175,6 +176,25 @@ def get_spent(conn, transaction_id, output): return (elem['block']['transactions'] for elem in cursor) +@register_query(MongoDBConnection) +def get_spending_transactions(conn, inputs): + cursor = conn.run( + conn.collection('bigchain').aggregate([ + {'$match': { + 'block.transactions.inputs.fulfills': { + '$in': inputs, + }, + }}, + {'$unwind': '$block.transactions'}, + {'$match': { + 'block.transactions.inputs.fulfills': { + '$in': inputs, + }, + }}, + ])) + return ((b['id'], b['block']['transactions']) for b in cursor) + + @register_query(MongoDBConnection) def get_owned_ids(conn, owner): cursor = conn.run( @@ -183,9 +203,7 @@ def get_owned_ids(conn, owner): {'$unwind': '$block.transactions'}, {'$match': {'block.transactions.outputs.public_keys': owner}} ])) - # we need to access some nested fields before returning so lets use a - # generator to avoid having to read all records on the cursor at this point - return (elem['block']['transactions'] for elem in cursor) + return ((b['id'], b['block']['transactions']) for b in cursor) @register_query(MongoDBConnection) @@ -196,6 +214,15 @@ def get_votes_by_block_id(conn, block_id): projection={'_id': False})) +@register_query(MongoDBConnection) +def get_votes_for_blocks_by_voter(conn, block_ids, node_pubkey): + return conn.run( + conn.collection('votes') + .find({'vote.voting_for_block': {'$in': block_ids}, + 'node_pubkey': node_pubkey}, + projection={'_id': False})) + + @register_query(MongoDBConnection) def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): return conn.run( @@ -206,10 +233,10 @@ def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): @register_query(MongoDBConnection) -def write_block(conn, block): +def write_block(conn, block_dict): return conn.run( conn.collection('bigchain') - .insert_one(block.to_dict())) + .insert_one(block_dict)) @register_query(MongoDBConnection) @@ -220,6 +247,31 @@ def get_block(conn, block_id): projection={'_id': False})) +@register_query(MongoDBConnection) +def write_assets(conn, assets): + try: + # unordered means that all the inserts will be attempted instead of + # stopping after the first error. + return conn.run( + conn.collection('assets') + .insert_many(assets, ordered=False)) + # This can happen if we try to write the same asset multiple times. + # One case is when we write the same transaction into multiple blocks due + # to invalid blocks. + # The actual mongodb exception is a BulkWriteError due to a duplicated key + # in one of the inserts. + except OperationError: + return + + +@register_query(MongoDBConnection) +def get_assets(conn, asset_ids): + return conn.run( + conn.collection('assets') + .find({'id': {'$in': asset_ids}}, + projection={'_id': False})) + + @register_query(MongoDBConnection) def count_blocks(conn): return conn.run( @@ -252,7 +304,7 @@ def get_genesis_block(conn): @register_query(MongoDBConnection) -def get_last_voted_block(conn, node_pubkey): +def get_last_voted_block_id(conn, node_pubkey): last_voted = conn.run( conn.collection('votes') .find({'node_pubkey': node_pubkey}, @@ -261,7 +313,7 @@ def get_last_voted_block(conn, node_pubkey): # pymongo seems to return a cursor even if there are no results # so we actually need to check the count if last_voted.count() == 0: - return get_genesis_block(conn) + return get_genesis_block(conn)['id'] mapping = {v['vote']['previous_block']: v['vote']['voting_for_block'] for v in last_voted} @@ -279,7 +331,7 @@ def get_last_voted_block(conn, node_pubkey): except KeyError: break - return get_block(conn, last_block_id) + return last_block_id @register_query(MongoDBConnection) @@ -301,3 +353,28 @@ def get_unvoted_blocks(conn, node_pubkey): 'votes': False, '_id': False }} ])) + + +@register_query(MongoDBConnection) +def text_search(conn, search, *, language='english', case_sensitive=False, + diacritic_sensitive=False, text_score=False, limit=0): + cursor = conn.run( + conn.collection('assets') + .find({'$text': { + '$search': search, + '$language': language, + '$caseSensitive': case_sensitive, + '$diacriticSensitive': diacritic_sensitive}}, + {'score': {'$meta': 'textScore'}, '_id': False}) + .sort([('score', {'$meta': 'textScore'})]) + .limit(limit)) + + if text_score: + return cursor + + return (_remove_text_score(asset) for asset in cursor) + + +def _remove_text_score(asset): + asset.pop('score', None) + return asset diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index ad89f9bc..6c54bfd8 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -2,7 +2,7 @@ import logging -from pymongo import ASCENDING, DESCENDING +from pymongo import ASCENDING, DESCENDING, TEXT from bigchaindb import backend from bigchaindb.common import exceptions @@ -27,7 +27,7 @@ def create_database(conn, dbname): @register_schema(MongoDBConnection) def create_tables(conn, dbname): - for table_name in ['bigchain', 'backlog', 'votes']: + for table_name in ['bigchain', 'backlog', 'votes', 'assets']: logger.info('Create `%s` table.', table_name) # create the table # TODO: read and write concerns can be declared here @@ -39,6 +39,7 @@ def create_indexes(conn, dbname): create_bigchain_secondary_index(conn, dbname) create_backlog_secondary_index(conn, dbname) create_votes_secondary_index(conn, dbname) + create_assets_secondary_index(conn, dbname) @register_schema(MongoDBConnection) @@ -102,3 +103,16 @@ def create_votes_secondary_index(conn, dbname): ASCENDING)], name='block_and_voter', unique=True) + + +def create_assets_secondary_index(conn, dbname): + logger.info('Create `assets` secondary index.') + + # unique index on the id of the asset. + # the id is the txid of the transaction that created the asset + conn.conn[dbname]['assets'].create_index('id', + name='asset_id', + unique=True) + + # full text search index + conn.conn[dbname]['assets'].create_index([('$**', TEXT)], name='text') diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 9aa653d7..f989c8b1 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -2,6 +2,8 @@ from functools import singledispatch +from bigchaindb.backend.exceptions import OperationError + @singledispatch def write_transaction(connection, signed_transaction): @@ -140,6 +142,20 @@ def get_spent(connection, transaction_id, condition_id): raise NotImplementedError +@singledispatch +def get_spending_transactions(connection, inputs): + """Return transactions which spend given inputs + + Args: + inputs (list): list of {txid, output} + + Returns: + Iterator of (block_ids, transaction) for transactions that + spend given inputs. + """ + raise NotImplementedError + + @singledispatch def get_owned_ids(connection, owner): """Retrieve a list of `txids` that can we used has inputs. @@ -148,9 +164,9 @@ def get_owned_ids(connection, owner): owner (str): base58 encoded public key. Returns: - A cursor for the matching transactions. + Iterator of (block_id, transaction) for transactions + that list given owner in conditions. """ - raise NotImplementedError @@ -183,6 +199,20 @@ def get_votes_by_block_id_and_voter(connection, block_id, node_pubkey): raise NotImplementedError +@singledispatch +def get_votes_for_blocks_by_voter(connection, block_ids, pubkey): + """Return votes for many block_ids + + Args: + block_ids (set): block_ids + pubkey (str): public key of voting node + + Returns: + A cursor of votes matching given block_ids and public key + """ + raise NotImplementedError + + @singledispatch def write_block(connection, block): """Write a block to the bigchain table. @@ -211,6 +241,33 @@ def get_block(connection, block_id): raise NotImplementedError +@singledispatch +def write_assets(connection, assets): + """Write a list of assets to the assets table. + + Args: + assets (list): a list of assets to write. + + Returns: + The database response. + """ + raise NotImplementedError + + +@singledispatch +def get_assets(connection, asset_ids): + """Get a list of assets from the assets table. + + Args: + asset_ids (list): a list of ids for the assets to be retrieved from + the database. + + Returns: + assets (list): the list of returned assets. + """ + raise NotImplementedError + + @singledispatch def count_blocks(connection): """Count the number of blocks in the bigchain table. @@ -259,15 +316,15 @@ def get_genesis_block(connection): @singledispatch -def get_last_voted_block(connection, node_pubkey): +def get_last_voted_block_id(connection, node_pubkey): """Get the last voted block for a specific node. Args: node_pubkey (str): base58 encoded public key. Returns: - The last block the node has voted on. If the node didn't cast - any vote then the genesis block is returned. + The id of the last block the node has voted on. If the node didn't cast + any vote then the genesis block id is returned. """ raise NotImplementedError @@ -298,3 +355,33 @@ def get_txids_filtered(connection, asset_id, operation=None): """ raise NotImplementedError + + +@singledispatch +def text_search(conn, search, *, language='english', case_sensitive=False, + diacritic_sensitive=False, text_score=False, limit=0): + """Return all the assets that match the text search. + + The results are sorted by text score. + For more information about the behavior of text search on MongoDB see + https://docs.mongodb.com/manual/reference/operator/query/text/#behavior + + Args: + search (str): Text search string to query the text index + language (str, optional): The language for the search and the rules for + stemmer and tokenizer. If the language is ``None`` text search uses + simple tokenization and no stemming. + case_sensitive (bool, optional): Enable or disable case sensitive + search. + diacritic_sensitive (bool, optional): Enable or disable case sensitive + diacritic search. + text_score (bool, optional): If ``True`` returns the text score with + each document. + limit (int, optional): Limit the number of returned documents. + + Returns: + :obj:`list` of :obj:`dict`: a list of assets + """ + + raise OperationError('This query is only supported when running ' + 'BigchainDB with MongoDB as the backend.') diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index 6011cc8c..923632fd 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -6,6 +6,7 @@ import rethinkdb as r from bigchaindb import backend, utils from bigchaindb.common import exceptions from bigchaindb.common.transaction import Transaction +from bigchaindb.common.utils import serialize from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection @@ -121,13 +122,14 @@ def get_spent(connection, transaction_id, output): @register_query(RethinkDBConnection) def get_owned_ids(connection, owner): - return connection.run( - r.table('bigchain', read_mode=READ_MODE) + query = (r.table('bigchain', read_mode=READ_MODE) .get_all(owner, index='outputs') .distinct() - .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda tx: tx['outputs'].contains( + .concat_map(unwind_block_transactions) + .filter(lambda doc: doc['tx']['outputs'].contains( lambda c: c['public_keys'].contains(owner)))) + cursor = connection.run(query) + return ((b['id'], b['tx']) for b in cursor) @register_query(RethinkDBConnection) @@ -147,10 +149,10 @@ def get_votes_by_block_id_and_voter(connection, block_id, node_pubkey): @register_query(RethinkDBConnection) -def write_block(connection, block): +def write_block(connection, block_dict): return connection.run( r.table('bigchain') - .insert(r.json(block.to_str()), durability=WRITE_DURABILITY)) + .insert(r.json(serialize(block_dict)), durability=WRITE_DURABILITY)) @register_query(RethinkDBConnection) @@ -158,6 +160,20 @@ def get_block(connection, block_id): return connection.run(r.table('bigchain').get(block_id)) +@register_query(RethinkDBConnection) +def write_assets(connection, assets): + return connection.run( + r.table('assets') + .insert(assets, durability=WRITE_DURABILITY)) + + +@register_query(RethinkDBConnection) +def get_assets(connection, asset_ids): + return connection.run( + r.table('assets', read_mode=READ_MODE) + .get_all(*asset_ids)) + + @register_query(RethinkDBConnection) def count_blocks(connection): return connection.run( @@ -188,7 +204,7 @@ def get_genesis_block(connection): @register_query(RethinkDBConnection) -def get_last_voted_block(connection, node_pubkey): +def get_last_voted_block_id(connection, node_pubkey): try: # get the latest value for the vote timestamp (over all votes) max_timestamp = connection.run( @@ -203,7 +219,7 @@ def get_last_voted_block(connection, node_pubkey): except r.ReqlNonExistenceError: # return last vote if last vote exists else return Genesis block - return get_genesis_block(connection) + return get_genesis_block(connection)['id'] # Now the fun starts. Since the resolution of timestamp is a second, # we might have more than one vote per timestamp. If this is the case @@ -235,9 +251,7 @@ def get_last_voted_block(connection, node_pubkey): except KeyError: break - return connection.run( - r.table('bigchain', read_mode=READ_MODE) - .get(last_block_id)) + return last_block_id @register_query(RethinkDBConnection) @@ -253,3 +267,30 @@ def get_unvoted_blocks(connection, node_pubkey): # database level. Solving issue #444 can help untangling the situation unvoted_blocks = filter(lambda block: not utils.is_genesis_block(block), unvoted) return unvoted_blocks + + +@register_query(RethinkDBConnection) +def get_votes_for_blocks_by_voter(connection, block_ids, node_pubkey): + return connection.run( + r.table('votes') + .filter(lambda row: r.expr(block_ids).contains(row['vote']['voting_for_block'])) + .filter(lambda row: row['node_pubkey'] == node_pubkey)) + + +def unwind_block_transactions(block): + """ Yield a block for each transaction in given block """ + return block['block']['transactions'].map(lambda tx: block.merge({'tx': tx})) + + +@register_query(RethinkDBConnection) +def get_spending_transactions(connection, links): + query = ( + r.table('bigchain') + .get_all(*[(l['txid'], l['output']) for l in links], index='inputs') + .concat_map(unwind_block_transactions) + # filter transactions spending output + .filter(lambda doc: r.expr(links).set_intersection( + doc['tx']['inputs'].map(lambda i: i['fulfills']))) + ) + cursor = connection.run(query) + return ((b['id'], b['tx']) for b in cursor) diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index 997ec5fc..8f0f6b9c 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -23,7 +23,7 @@ def create_database(connection, dbname): @register_schema(RethinkDBConnection) def create_tables(connection, dbname): - for table_name in ['bigchain', 'backlog', 'votes']: + for table_name in ['bigchain', 'backlog', 'votes', 'assets']: logger.info('Create `%s` table.', table_name) connection.run(r.db(dbname).table_create(table_name)) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 6e271d60..365fee8f 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -161,6 +161,9 @@ class TransactionLink(object): # TODO: If `other !== TransactionLink` return `False` return self.to_dict() == other.to_dict() + def __hash__(self): + return hash((self.txid, self.output)) + @classmethod def from_dict(cls, link): """Transforms a Python dictionary to a TransactionLink object. diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 5d2e9c03..923cb8a8 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -4,11 +4,10 @@ from time import time from bigchaindb import exceptions as core_exceptions from bigchaindb.common import crypto, exceptions from bigchaindb.common.utils import gen_timestamp, serialize -from bigchaindb.common.transaction import TransactionLink import bigchaindb -from bigchaindb import backend, config_utils, utils +from bigchaindb import backend, config_utils, fastquery from bigchaindb.consensus import BaseConsensusRules from bigchaindb.models import Block, Transaction @@ -183,15 +182,23 @@ class Bigchain(object): include_status (bool): also return the status of the block the return value is then a tuple: (block, status) """ - block = backend.query.get_block(self.connection, block_id) - status = None + # get block from database + block_dict = backend.query.get_block(self.connection, block_id) + # get the asset ids from the block + if block_dict: + asset_ids = Block.get_asset_ids(block_dict) + # get the assets from the database + assets = self.get_assets(asset_ids) + # add the assets to the block transactions + block_dict = Block.couple_assets(block_dict, assets) + status = None if include_status: - if block: - status = self.block_election_status(block) - return block, status + if block_dict: + status = self.block_election_status(block_dict) + return block_dict, status else: - return block + return block_dict def get_transaction(self, txid, include_status=False): """Get the transaction with the specified `txid` (and optionally its status) @@ -251,7 +258,13 @@ class Bigchain(object): tx_status = self.TX_IN_BACKLOG if response: - response = Transaction.from_dict(response) + if tx_status == self.TX_IN_BACKLOG: + response = Transaction.from_dict(response) + else: + # If we are reading from the bigchain collection the asset is + # not in the transaction so we need to fetch the asset and + # reconstruct the transaction. + response = Transaction.from_db(self, response) if include_status: return response, tx_status @@ -376,51 +389,6 @@ class Bigchain(object): # Either no transaction was returned spending the `(txid, output)` as # input or the returned transactions are not valid. - def get_outputs(self, owner): - """Retrieve a list of links to transaction outputs for a given public - key. - - Args: - owner (str): base58 encoded public key. - - Returns: - :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s - pointing to another transaction's condition - """ - # get all transactions in which owner is in the `owners_after` list - response = backend.query.get_owned_ids(self.connection, owner) - return [ - TransactionLink(tx['id'], index) - for tx in response - if not self.is_tx_strictly_in_invalid_block(tx['id']) - for index, output in enumerate(tx['outputs']) - if utils.output_has_owner(output, owner) - ] - - def is_tx_strictly_in_invalid_block(self, txid): - """ - Checks whether the transaction with the given ``txid`` - *strictly* belongs to an invalid block. - - Args: - txid (str): Transaction id. - - Returns: - bool: ``True`` if the transaction *strictly* belongs to a - block that is invalid. ``False`` otherwise. - - Note: - Since a transaction may be in multiple blocks, with - different statuses, the term "strictly" is used to - emphasize that if a transaction is said to be in an invalid - block, it means that it is not in any other block that is - either valid or undecided. - - """ - validity = self.get_blocks_status_containing_tx(txid) - return (Bigchain.BLOCK_VALID not in validity.values() and - Bigchain.BLOCK_UNDECIDED not in validity.values()) - def get_owned_ids(self, owner): """Retrieve a list of ``txid`` s that can be used as inputs. @@ -433,14 +401,17 @@ class Bigchain(object): """ return self.get_outputs_filtered(owner, include_spent=False) + @property + def fastquery(self): + return fastquery.FastQuery(self.connection, self.me) + def get_outputs_filtered(self, owner, include_spent=True): """ Get a list of output links filtered on some criteria """ - outputs = self.get_outputs(owner) + outputs = self.fastquery.get_outputs_by_public_key(owner) if not include_spent: - outputs = [o for o in outputs - if not self.get_spent(o.txid, o.output)] + outputs = self.fastquery.filter_spent_outputs(outputs) return outputs def get_transactions_filtered(self, asset_id, operation=None): @@ -513,7 +484,14 @@ class Bigchain(object): block (Block): block to write to bigchain. """ - return backend.query.write_block(self.connection, block) + # Decouple assets from block + assets, block_dict = block.decouple_assets() + # write the assets + if assets: + self.write_assets(assets) + + # write the block + return backend.query.write_block(self.connection, block_dict) def prepare_genesis_block(self): """Prepare a genesis block.""" @@ -592,7 +570,9 @@ class Bigchain(object): def get_last_voted_block(self): """Returns the last block that this node voted on.""" - return Block.from_dict(backend.query.get_last_voted_block(self.connection, self.me)) + last_block_id = backend.query.get_last_voted_block_id(self.connection, + self.me) + return Block.from_dict(self.get_block(last_block_id)) def get_unvoted_blocks(self): """Return all the blocks that have not been voted on by this node. @@ -616,3 +596,37 @@ class Bigchain(object): """Tally the votes on a block, and return the status: valid, invalid, or undecided.""" return self.block_election(block)['status'] + + def get_assets(self, asset_ids): + """ + Return a list of assets that match the asset_ids + + Args: + asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to + retrieve from the database. + + Returns: + list: The list of assets returned from the database. + """ + return backend.query.get_assets(self.connection, asset_ids) + + def write_assets(self, assets): + """ + Writes a list of assets into the database. + + Args: + assets (:obj:`list` of :obj:`dict`): A list of assets to write to + the database. + """ + return backend.query.write_assets(self.connection, assets) + + def text_search(self, search, *, limit=0): + assets = backend.query.text_search(self.connection, search, limit=limit) + + # TODO: This is not efficient. There may be a more efficient way to + # query by storing block ids with the assets and using fastquery. + # See https://github.com/bigchaindb/bigchaindb/issues/1496 + for asset in assets: + tx, status = self.get_transaction(asset['id'], True) + if status == self.TX_VALID: + yield asset diff --git a/bigchaindb/fastquery.py b/bigchaindb/fastquery.py new file mode 100644 index 00000000..d19294ce --- /dev/null +++ b/bigchaindb/fastquery.py @@ -0,0 +1,70 @@ +from bigchaindb.utils import output_has_owner +from bigchaindb.backend import query +from bigchaindb.common.transaction import TransactionLink + + +class FastQuery: + """ + Database queries that join on block results from a single node. + + * Votes are not validated for security (security is a replication concern) + * Votes come from only one node, and as such, non-byzantine fault tolerance + is reduced. + + Previously, to consider the status of a block, all votes for that block + were retrieved and the election results were counted. This meant that a + faulty node may still have been able to obtain a correct election result. + However, from the point of view of a client, it is still neccesary to + query multiple nodes to insure against getting an incorrect response from + a byzantine node. + """ + def __init__(self, connection, me): + self.connection = connection + self.me = me + + def filter_valid_block_ids(self, block_ids, include_undecided=False): + """ + Given block ids, return only the ones that are valid. + """ + block_ids = list(set(block_ids)) + votes = query.get_votes_for_blocks_by_voter( + self.connection, block_ids, self.me) + votes = {vote['vote']['voting_for_block']: vote['vote']['is_block_valid'] + for vote in votes} + return [block_id for block_id in block_ids + if votes.get(block_id, include_undecided)] + + def filter_valid_items(self, items, block_id_key=lambda b: b[0]): + """ + Given items with block ids, return only the ones that are valid or undecided. + """ + items = list(items) + block_ids = map(block_id_key, items) + valid_block_ids = set(self.filter_valid_block_ids(block_ids, True)) + return [b for b in items if block_id_key(b) in valid_block_ids] + + def get_outputs_by_public_key(self, public_key): + """ + Get outputs for a public key + """ + res = list(query.get_owned_ids(self.connection, public_key)) + txs = [tx for _, tx in self.filter_valid_items(res)] + return [TransactionLink(tx['id'], index) + for tx in txs + for index, output in enumerate(tx['outputs']) + if output_has_owner(output, public_key)] + + def filter_spent_outputs(self, outputs): + """ + Remove outputs that have been spent + + Args: + outputs: list of TransactionLink + """ + links = [o.to_dict() for o in outputs] + res = query.get_spending_transactions(self.connection, links) + txs = [tx for _, tx in self.filter_valid_items(res)] + spends = {TransactionLink.from_dict(input_['fulfills']) + for tx in txs + for input_ in tx['inputs']} + return [ff for ff in outputs if ff not in spends] diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 66c267a9..c8ad9dd3 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -1,3 +1,5 @@ +from copy import deepcopy + from bigchaindb.common.crypto import hash_data, PublicKey, PrivateKey from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, DoubleSpend, InputDoesNotExist, @@ -84,6 +86,31 @@ class Transaction(Transaction): validate_transaction_schema(tx_body) return super().from_dict(tx_body) + @classmethod + def from_db(cls, bigchain, tx_dict): + """ + Helper method that reconstructs a transaction dict that was returned + from the database. It checks what asset_id to retrieve, retrieves the + asset from the asset table and reconstructs the transaction. + + Args: + bigchain (:class:`~bigchaindb.Bigchain`): An instance of Bigchain + used to perform database queries. + tx_dict (:obj:`dict`): The transaction dict as returned from the + database. + + Returns: + :class:`~Transaction` + + """ + if tx_dict['operation'] in [Transaction.CREATE, Transaction.GENESIS]: + # TODO: Maybe replace this call to a call to get_asset_by_id + asset = list(bigchain.get_assets([tx_dict['id']]))[0] + del asset['id'] + tx_dict.update({'asset': asset}) + + return cls.from_dict(tx_dict) + class Block(object): """Bundle a list of Transactions in a Block. Nodes vote on its validity. @@ -300,6 +327,98 @@ class Block(object): 'signature': self.signature, } + @classmethod + def from_db(cls, bigchain, block_dict, from_dict_kwargs=None): + """ + Helper method that reconstructs a block_dict that was returned from + the database. It checks what asset_ids to retrieve, retrieves the + assets from the assets table and reconstructs the block. + + Args: + bigchain (:class:`~bigchaindb.Bigchain`): An instance of Bigchain + used to perform database queries. + block_dict(:obj:`dict`): The block dict as returned from the + database. + from_dict_kwargs (:obj:`dict`): additional kwargs to pass to from_dict + + Returns: + :class:`~Block` + + """ + asset_ids = cls.get_asset_ids(block_dict) + assets = bigchain.get_assets(asset_ids) + block_dict = cls.couple_assets(block_dict, assets) + kwargs = from_dict_kwargs or {} + return cls.from_dict(block_dict, **kwargs) + + def decouple_assets(self): + """ + Extracts the assets from the ``CREATE`` transactions in the block. + + Returns: + tuple: (assets, block) with the assets being a list of dicts and + the block being the dict of the block with no assets in the CREATE + transactions. + """ + block_dict = deepcopy(self.to_dict()) + assets = [] + for transaction in block_dict['block']['transactions']: + if transaction['operation'] in [Transaction.CREATE, + Transaction.GENESIS]: + asset = transaction.pop('asset') + asset.update({'id': transaction['id']}) + assets.append(asset) + + return (assets, block_dict) + + @staticmethod + def couple_assets(block_dict, assets): + """ + Given a block_dict with no assets (as returned from a database call) + and a list of assets, reconstruct the original block by putting the + assets back into the ``CREATE`` transactions in the block. + + Args: + block_dict (:obj:`dict`): The block dict as returned from a + database call. + assets (:obj:`list` of :obj:`dict`): A list of assets returned from + a database call. + + Returns: + dict: The dict of the reconstructed block. + """ + # create a dict with {'': asset} + assets = {asset.pop('id'): asset for asset in assets} + # add the assets to the block transactions + for transaction in block_dict['block']['transactions']: + if transaction['operation'] in [Transaction.CREATE, + Transaction.GENESIS]: + transaction.update({'asset': assets.get(transaction['id'])}) + return block_dict + + @staticmethod + def get_asset_ids(block_dict): + """ + Given a block_dict return all the asset_ids for that block (the txid + of CREATE transactions). Useful to know which assets to retrieve + from the database to reconstruct the block. + + Args: + block_dict (:obj:`dict`): The block dict as returned from a + database call. + + Returns: + list: The list of asset_ids in the block. + + """ + asset_ids = [] + for transaction in block_dict['block']['transactions']: + if transaction['operation'] in [Transaction.CREATE, + Transaction.GENESIS]: + asset_ids.append(transaction['id']) + + return asset_ids + def to_str(self): return serialize(self.to_dict()) diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index dc516845..bdd9e0f1 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -51,7 +51,9 @@ class Vote: def validate_block(self, block_dict): if not self.bigchain.has_previous_vote(block_dict['id']): try: - block = Block.from_dict(block_dict, tx_construct=FastTransaction) + block = Block.from_db(self.bigchain, block_dict, from_dict_kwargs={ + 'tx_construct': FastTransaction + }) except (exceptions.InvalidHash): # XXX: if a block is invalid we should skip the `validate_tx` # step, but since we are in a pipeline we cannot just jump to diff --git a/docs/server/source/cloud-deployment-templates/azure-quickstart-template.md b/docs/server/source/appendices/azure-quickstart-template.md similarity index 93% rename from docs/server/source/cloud-deployment-templates/azure-quickstart-template.md rename to docs/server/source/appendices/azure-quickstart-template.md index 1bf35a31..59f52fd3 100644 --- a/docs/server/source/cloud-deployment-templates/azure-quickstart-template.md +++ b/docs/server/source/appendices/azure-quickstart-template.md @@ -1,6 +1,6 @@ # Azure Quickstart Template -If you didn't read the introduction to the [cloud deployment templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. +This page outlines how to run a single BigchainDB node on the Microsoft Azure public cloud, with RethinkDB as the database backend. It uses an Azure Quickstart Template. That template is dated because we now recommend using MongoDB instead of RethinkDB. That's why we moved this page to the Appendices. Note: There was an Azure quickstart template in the `blockchain` directory of Microsoft's `Azure/azure-quickstart-templates` repository on GitHub. It's gone now; it was replaced by the one described here. diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 1c969c05..f5931e64 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -18,6 +18,9 @@ Appendices backend commands aws-setup + template-terraform-aws + template-ansible + azure-quickstart-template generate-key-pair-for-ssh firewall-notes ntp-notes diff --git a/docs/server/source/cloud-deployment-templates/template-ansible.md b/docs/server/source/appendices/template-ansible.md similarity index 94% rename from docs/server/source/cloud-deployment-templates/template-ansible.md rename to docs/server/source/appendices/template-ansible.md index f296a2cf..508d0555 100644 --- a/docs/server/source/cloud-deployment-templates/template-ansible.md +++ b/docs/server/source/appendices/template-ansible.md @@ -1,9 +1,9 @@ # Template: Ansible Playbook to Run a BigchainDB Node on an Ubuntu Machine -If you didn't read the introduction to the [cloud deployment templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. - This page explains how to use [Ansible](https://www.ansible.com/) to install, configure and run all the software needed to run a one-machine BigchainDB node on a server running Ubuntu 16.04. +**Note: We're not actively maintaining the associated Ansible files (e.g. playbooks). They are RethinkDB-specific, even though we now recommend using MongoDB. You may find the old Ansible stuff useful nevertheless, which is why we moved this page to the Appendices rather than deleting it.** + ## Install Ansible diff --git a/docs/server/source/cloud-deployment-templates/template-terraform-aws.md b/docs/server/source/appendices/template-terraform-aws.md similarity index 95% rename from docs/server/source/cloud-deployment-templates/template-terraform-aws.md rename to docs/server/source/appendices/template-terraform-aws.md index d4a22e83..055a5ee3 100644 --- a/docs/server/source/cloud-deployment-templates/template-terraform-aws.md +++ b/docs/server/source/appendices/template-terraform-aws.md @@ -1,8 +1,8 @@ # Template: Using Terraform to Provision an Ubuntu Machine on AWS -If you didn't read the introduction to the [cloud deployment templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. +This page explains a way to use [Terraform](https://www.terraform.io/) to provision an Ubuntu machine (i.e. an EC2 instance with Ubuntu 16.04) and other resources on [AWS](https://aws.amazon.com/). That machine can then be used to host a one-machine BigchainDB node, for example. -This page explains a way to use [Terraform](https://www.terraform.io/) to provision an Ubuntu machine (i.e. an EC2 instance with Ubuntu 16.04) and other resources on [AWS](https://aws.amazon.com/). That machine can then be used to host a one-machine BigchainDB node. +**Note: We're not actively maintaining the associated Terraform files. You may find them useful nevertheless, which is why we moved this page to the Appendices rather than deleting it.** ## Install Terraform diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index af2dc743..a94fab94 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -1,11 +1,14 @@ -Cloud Deployment Templates -========================== +Production Deployment Template +============================== -We have some "templates" to deploy a basic, working, but bare-bones BigchainDB node on various cloud providers. They should *not* be used as-is to deploy a node for production. They can be used as a starting point. +This section outlines how *we* deploy production BigchainDB nodes and clusters +on Microsoft Azure +using Kubernetes. +We improve it constantly. +You may choose to use it as a template or reference for your own deployment, +but *we make no claim that it is suitable for your purposes*. +Feel free change things to suit your needs or preferences. -You don't have to use the tools we use in the templates. You can use whatever tools you prefer. - -If you find the cloud deployment templates for nodes helpful, then you may also be interested in our scripts for :doc:`deploying a testing cluster on AWS <../clusters-feds/aws-testing-cluster>` (documented in the Clusters section). .. toctree:: :maxdepth: 1 @@ -15,9 +18,6 @@ If you find the cloud deployment templates for nodes helpful, then you may also server-tls-certificate client-tls-certificate revoke-tls-certificate - template-terraform-aws - template-ansible - azure-quickstart-template template-kubernetes-azure node-on-kubernetes add-node-on-kubernetes diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 0bfde7ad..ef749d55 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -20,7 +20,7 @@ Community-Driven Libraries and Tools Some of these projects are a work in progress, but may still be useful. -* `Javascript transaction builder `_ +* `JavaScript / Node.js driver `_ * `Haskell transaction builder `_ * `Go driver `_ * `Java driver `_ diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index bf45aca0..0dee9174 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -7,14 +7,14 @@ BigchainDB Server Documentation ← Back to All BigchainDB Docs introduction quickstart - cloud-deployment-templates/index production-nodes/index + clusters-feds/index + cloud-deployment-templates/index dev-and-test/index server-reference/index http-client-server-api websocket-event-stream-api drivers-clients/index - clusters-feds/index data-models/index schema/transaction schema/vote diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 2eae07f0..5c2b0500 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -8,7 +8,7 @@ A. Install MongoDB as the database backend. (There are other options but you can B. Run MongoDB. Open a Terminal and run the command: ```text -$ mongod --replSet=bigchain-rs +$ sudo mongod --replSet=bigchain-rs ``` C. Ubuntu 16.04 already has Python 3.5, so you don't need to install it, but you do need to install some other things: diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 1e8ca56d..140ef50e 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:0.10.1 + image: bigchaindb/bigchaindb:0.10.2 imagePullPolicy: IfNotPresent args: - start diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index aa3a9bca..4fe5618f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: bigchaindb -version: master +version: git summary: a scalable blockchain database description: | With high throughput, sub-second latency and powerful functionality to diff --git a/tests/README.md b/tests/README.md index d0e2da52..e6de82c9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,8 +27,7 @@ BigchainDB from source. The [`CONTRIBUTING.md` file](../CONTRIBUTING.md) has instructions for how to do that. Next, make sure you have RethinkDB or MongoDB running in the background. You -can run RethinkDB using `rethinkdb --daemon` or MongoDB using `mongod ---replSet=rs0`. +can run RethinkDB using `rethinkdb --daemon` or MongoDB using `mongod --replSet=bigchain-rs`. The `pytest` command has many options. If you want to learn about all the things you can do with pytest, see [the pytest diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index bd7e75f1..897a0f06 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -1,4 +1,7 @@ +from copy import deepcopy + import pytest +import pymongo pytestmark = pytest.mark.bdb @@ -205,10 +208,10 @@ def test_get_owned_ids(signed_create_tx, user_pk): block = Block(transactions=[signed_create_tx]) conn.db.bigchain.insert_one(block.to_dict()) - owned_ids = list(query.get_owned_ids(conn, user_pk)) + [(block_id, tx)] = list(query.get_owned_ids(conn, user_pk)) - assert len(owned_ids) == 1 - assert owned_ids[0] == signed_create_tx.to_dict() + assert block_id == block.id + assert tx == signed_create_tx.to_dict() def test_get_votes_by_block_id(signed_create_tx, structurally_valid_vote): @@ -269,7 +272,7 @@ def test_write_block(signed_create_tx): # create and write block block = Block(transactions=[signed_create_tx]) - query.write_block(conn, block) + query.write_block(conn, block.to_dict()) block_db = conn.db.bigchain.find_one({'id': block.id}, {'_id': False}) @@ -347,17 +350,18 @@ def test_get_genesis_block(genesis_block): from bigchaindb.backend import connect, query conn = connect() - assert query.get_genesis_block(conn) == genesis_block.to_dict() + assets, genesis_block_dict = genesis_block.decouple_assets() + assert query.get_genesis_block(conn) == genesis_block_dict -def test_get_last_voted_block(genesis_block, signed_create_tx, b): +def test_get_last_voted_block_id(genesis_block, signed_create_tx, b): from bigchaindb.backend import connect, query from bigchaindb.models import Block from bigchaindb.common.exceptions import CyclicBlockchainError conn = connect() # check that the last voted block is the genesis block - assert query.get_last_voted_block(conn, b.me) == genesis_block.to_dict() + assert query.get_last_voted_block_id(conn, b.me) == genesis_block.id # create and insert a new vote and block block = Block(transactions=[signed_create_tx]) @@ -365,7 +369,7 @@ def test_get_last_voted_block(genesis_block, signed_create_tx, b): vote = b.vote(block.id, genesis_block.id, True) conn.db.votes.insert_one(vote) - assert query.get_last_voted_block(conn, b.me) == block.to_dict() + assert query.get_last_voted_block_id(conn, b.me) == block.id # force a bad chain vote.pop('_id') @@ -374,7 +378,7 @@ def test_get_last_voted_block(genesis_block, signed_create_tx, b): conn.db.votes.insert_one(vote) with pytest.raises(CyclicBlockchainError): - query.get_last_voted_block(conn, b.me) + query.get_last_voted_block_id(conn, b.me) def test_get_unvoted_blocks(signed_create_tx): @@ -417,3 +421,179 @@ def test_get_txids_filtered(signed_create_tx, signed_transfer_tx): # Test get by asset and TRANSFER txids = set(query.get_txids_filtered(conn, asset_id, Transaction.TRANSFER)) assert txids == {signed_transfer_tx.id} + + +def test_get_spending_transactions(user_pk): + from bigchaindb.backend import connect, query + from bigchaindb.models import Block, Transaction + conn = connect() + + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 3) + inputs = tx1.to_inputs() + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx3 = Transaction.transfer([inputs[1]], out, tx1.id) + tx4 = Transaction.transfer([inputs[2]], out, tx1.id) + block = Block([tx1, tx2, tx3, tx4]) + conn.db.bigchain.insert_one(block.to_dict()) + + links = [inputs[0].fulfills.to_dict(), inputs[2].fulfills.to_dict()] + res = list(query.get_spending_transactions(conn, links)) + + # tx3 not a member because input 1 not asked for + assert res == [(block.id, tx2.to_dict()), (block.id, tx4.to_dict())] + + +def test_get_votes_for_blocks_by_voter(): + from bigchaindb.backend import connect, query + + conn = connect() + votes = [ + { + 'node_pubkey': 'a', + 'vote': {'voting_for_block': 'block1'}, + }, + { + 'node_pubkey': 'b', + 'vote': {'voting_for_block': 'block1'}, + }, + { + 'node_pubkey': 'a', + 'vote': {'voting_for_block': 'block2'}, + }, + { + 'node_pubkey': 'a', + 'vote': {'voting_for_block': 'block3'}, + } + ] + for vote in votes: + conn.db.votes.insert_one(vote.copy()) + res = query.get_votes_for_blocks_by_voter(conn, ['block1', 'block2'], 'a') + assert list(res) == [votes[0], votes[2]] + + +def test_write_assets(): + from bigchaindb.backend import connect, query + conn = connect() + + assets = [ + {'id': 1, 'data': '1'}, + {'id': 2, 'data': '2'}, + {'id': 3, 'data': '3'}, + # Duplicated id. Should not be written to the database + {'id': 1, 'data': '1'}, + ] + + # write the assets + query.write_assets(conn, deepcopy(assets)) + + # check that 3 assets were written to the database + cursor = conn.db.assets.find({}, projection={'_id': False})\ + .sort('id', pymongo.ASCENDING) + + assert cursor.count() == 3 + assert list(cursor) == assets[:-1] + + +def test_get_assets(): + from bigchaindb.backend import connect, query + conn = connect() + + assets = [ + {'id': 1, 'data': '1'}, + {'id': 2, 'data': '2'}, + {'id': 3, 'data': '3'}, + ] + + # write the assets + conn.db.assets.insert_many(deepcopy(assets), ordered=False) + + # read only 2 assets + cursor = query.get_assets(conn, [1, 3]) + + assert cursor.count() == 2 + assert list(cursor.sort('id', pymongo.ASCENDING)) == assets[::2] + + +def test_text_search(): + from bigchaindb.backend import connect, query + conn = connect() + + # Example data and tests cases taken from the mongodb documentation + # https://docs.mongodb.com/manual/reference/operator/query/text/ + assets = [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + {'id': 3, 'subject': 'Baking a cake', 'author': 'abc', 'views': 90}, + {'id': 4, 'subject': 'baking', 'author': 'xyz', 'views': 100}, + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + {'id': 6, 'subject': 'Сырники', 'author': 'jkl', 'views': 80}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, + {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} + ] + + # insert the assets + conn.db.assets.insert_many(deepcopy(assets), ordered=False) + + # test search single word + assert list(query.text_search(conn, 'coffee')) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, + ] + + # match any of the search terms + assert list(query.text_search(conn, 'bake coffee cake')) == [ + {'author': 'abc', 'id': 3, 'subject': 'Baking a cake', 'views': 90}, + {'author': 'xyz', 'id': 1, 'subject': 'coffee', 'views': 50}, + {'author': 'xyz', 'id': 4, 'subject': 'baking', 'views': 100}, + {'author': 'efg', 'id': 2, 'subject': 'Coffee Shopping', 'views': 5}, + {'author': 'efg', 'id': 7, 'subject': 'coffee and cream', 'views': 10} + ] + + # search for a phrase + assert list(query.text_search(conn, '\"coffee shop\"')) == [ + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + ] + + # exclude documents that contain a term + assert list(query.text_search(conn, 'coffee -shop')) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, + ] + + # search different language + assert list(query.text_search(conn, 'leche', language='es')) == [ + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} + ] + + # case and diacritic insensitive search + assert list(query.text_search(conn, 'сы́рники CAFÉS')) == [ + {'id': 6, 'subject': 'Сырники', 'author': 'jkl', 'views': 80}, + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} + ] + + # case sensitive search + assert list(query.text_search(conn, 'Coffee', case_sensitive=True)) == [ + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + ] + + # diacritic sensitive search + assert list(query.text_search(conn, 'CAFÉ', diacritic_sensitive=True)) == [ + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + ] + + # return text score + assert list(query.text_search(conn, 'coffee', text_score=True)) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50, 'score': 1.0}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5, 'score': 0.75}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10, 'score': 0.75}, + ] + + # limit search result + assert list(query.text_search(conn, 'coffee', limit=2)) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + ] diff --git a/tests/backend/mongodb/test_schema.py b/tests/backend/mongodb/test_schema.py index 71eac7ff..e11dbfe8 100644 --- a/tests/backend/mongodb/test_schema.py +++ b/tests/backend/mongodb/test_schema.py @@ -18,7 +18,8 @@ def test_init_creates_db_tables_and_indexes(): init_database() collection_names = conn.conn[dbname].collection_names() - assert sorted(collection_names) == ['backlog', 'bigchain', 'votes'] + assert sorted(collection_names) == ['assets', 'backlog', 'bigchain', + 'votes'] indexes = conn.conn[dbname]['bigchain'].index_information().keys() assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', @@ -31,6 +32,9 @@ def test_init_creates_db_tables_and_indexes(): indexes = conn.conn[dbname]['votes'].index_information().keys() assert sorted(indexes) == ['_id_', 'block_and_voter'] + indexes = conn.conn[dbname]['assets'].index_information().keys() + assert sorted(indexes) == ['_id_', 'asset_id', 'text'] + def test_init_database_fails_if_db_exists(): import bigchaindb @@ -62,7 +66,8 @@ def test_create_tables(): schema.create_tables(conn, dbname) collection_names = conn.conn[dbname].collection_names() - assert sorted(collection_names) == ['backlog', 'bigchain', 'votes'] + assert sorted(collection_names) == ['assets', 'backlog', 'bigchain', + 'votes'] def test_create_secondary_indexes(): diff --git a/tests/backend/rethinkdb/test_schema.py b/tests/backend/rethinkdb/test_schema.py index e19dfdc2..6f77b672 100644 --- a/tests/backend/rethinkdb/test_schema.py +++ b/tests/backend/rethinkdb/test_schema.py @@ -63,7 +63,8 @@ def test_create_tables(): assert conn.run(r.db(dbname).table_list().contains('bigchain')) is True assert conn.run(r.db(dbname).table_list().contains('backlog')) is True assert conn.run(r.db(dbname).table_list().contains('votes')) is True - assert len(conn.run(r.db(dbname).table_list())) == 3 + assert conn.run(r.db(dbname).table_list().contains('assets')) is True + assert len(conn.run(r.db(dbname).table_list())) == 4 @pytest.mark.bdb diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 57a644ee..4a7cd5ec 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -30,12 +30,16 @@ def test_schema(schema_func_name, args_qty): ('write_block', 1), ('get_block', 1), ('write_vote', 1), - ('get_last_voted_block', 1), + ('get_last_voted_block_id', 1), ('get_unvoted_blocks', 1), ('get_spent', 2), ('get_votes_by_block_id_and_voter', 2), ('update_transaction', 2), ('get_transaction_from_block', 2), + ('get_votes_for_blocks_by_voter', 2), + ('get_spending_transactions', 1), + ('write_assets', 1), + ('get_assets', 1), )) def test_query(query_func_name, args_qty): from bigchaindb.backend import query diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 3f05385c..05b07bf6 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -213,6 +213,88 @@ class TestBigchainApi(object): assert b.get_transaction(tx1.id) is None assert b.get_transaction(tx2.id) == tx2 + @pytest.mark.genesis + def test_text_search(self, b): + from bigchaindb.models import Transaction + from bigchaindb.backend.exceptions import OperationError + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + # define the assets + asset1 = {'msg': 'BigchainDB 1'} + asset2 = {'msg': 'BigchainDB 2'} + asset3 = {'msg': 'BigchainDB 3'} + + # create the transactions + tx1 = Transaction.create([b.me], [([b.me], 1)], + asset=asset1).sign([b.me_private]) + tx2 = Transaction.create([b.me], [([b.me], 1)], + asset=asset2).sign([b.me_private]) + tx3 = Transaction.create([b.me], [([b.me], 1)], + asset=asset3).sign([b.me_private]) + + # create the block + block = b.create_block([tx1, tx2, tx3]) + b.write_block(block) + + # vote valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + + # get the assets through text search + # this query only works with MongoDB + try: + assets = list(b.text_search('bigchaindb')) + except OperationError as exc: + assert not isinstance(b.connection, MongoDBConnection) + else: + assert len(assets) == 3 + + @pytest.mark.genesis + def test_text_search_returns_valid_only(self, monkeypatch, b): + from bigchaindb.models import Transaction + from bigchaindb.backend.exceptions import OperationError + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + asset_valid = {'msg': 'Hello BigchainDB!'} + asset_invalid = {'msg': 'Goodbye BigchainDB!'} + + monkeypatch.setattr('time.time', lambda: 1000000000) + tx1 = Transaction.create([b.me], [([b.me], 1)], + asset=asset_valid) + tx1 = tx1.sign([b.me_private]) + block1 = b.create_block([tx1]) + b.write_block(block1) + + monkeypatch.setattr('time.time', lambda: 1000000020) + tx2 = Transaction.create([b.me], [([b.me], 1)], + asset=asset_invalid) + tx2 = tx2.sign([b.me_private]) + block2 = b.create_block([tx2]) + b.write_block(block2) + + # vote the first block valid + vote = b.vote(block1.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + + # vote the second block invalid + vote = b.vote(block2.id, b.get_last_voted_block().id, False) + b.write_vote(vote) + + # get assets with text search + try: + assets = list(b.text_search('bigchaindb')) + except OperationError: + assert not isinstance(b.connection, MongoDBConnection) + return + + # should only return one asset + assert len(assets) == 1 + # should return the asset created by tx1 + assert assets[0] == { + 'data': {'msg': 'Hello BigchainDB!'}, + 'id': tx1.id + } + @pytest.mark.usefixtures('inputs') def test_write_transaction(self, b, user_pk, user_sk): from bigchaindb import Bigchain @@ -383,7 +465,7 @@ class TestBigchainApi(object): from bigchaindb.backend import query genesis = query.get_genesis_block(b.connection) - genesis = Block.from_dict(genesis) + genesis = Block.from_db(b, genesis) gb = b.get_last_voted_block() assert gb == genesis assert b.validate_block(gb) == gb @@ -1119,11 +1201,11 @@ def test_get_owned_ids_calls_get_outputs_filtered(): def test_get_outputs_filtered_only_unspent(): from bigchaindb.common.transaction import TransactionLink from bigchaindb.core import Bigchain - with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: + with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: - get_spent.side_effect = [True, False] + with patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') as filter_spent: + filter_spent.return_value = [TransactionLink('b', 2)] out = Bigchain().get_outputs_filtered('abc', include_spent=False) get_outputs.assert_called_once_with('abc') assert out == [TransactionLink('b', 2)] @@ -1132,13 +1214,13 @@ def test_get_outputs_filtered_only_unspent(): def test_get_outputs_filtered(): from bigchaindb.common.transaction import TransactionLink from bigchaindb.core import Bigchain - with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: + with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: + with patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') as filter_spent: out = Bigchain().get_outputs_filtered('abc') get_outputs.assert_called_once_with('abc') - get_spent.assert_not_called() + filter_spent.assert_not_called() assert out == get_outputs.return_value diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index 58c0eae4..8ade412e 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -20,6 +20,15 @@ def dummy_block(b): return block +def decouple_assets(b, block): + # the block comming from the database does not contain the assets + # so we need to pass the block without the assets and store the assets + # so that the voting pipeline can reconstruct it + assets, block_dict = block.decouple_assets() + b.write_assets(assets) + return block_dict + + DUMMY_SHA3 = '0123456789abcdef' * 4 @@ -79,9 +88,10 @@ def test_vote_validate_block(b): tx = dummy_tx(b) block = b.create_block([tx]) + block_dict = decouple_assets(b, block) vote_obj = vote.Vote() - validation = vote_obj.validate_block(block.to_dict()) + validation = vote_obj.validate_block(block_dict) assert validation[0] == block.id for tx1, tx2 in zip(validation[1], block.transactions): assert tx1 == tx2.to_dict() @@ -219,8 +229,9 @@ def test_valid_block_voting_multiprocessing(b, genesis_block, monkeypatch): vote_pipeline.setup(indata=inpipe, outdata=outpipe) block = dummy_block(b) + block_dict = decouple_assets(b, block) - inpipe.put(block.to_dict()) + inpipe.put(block_dict) vote_pipeline.start() vote_out = outpipe.get() vote_pipeline.terminate() @@ -256,6 +267,7 @@ def test_valid_block_voting_with_create_transaction(b, monkeypatch.setattr('time.time', lambda: 1111111111) block = b.create_block([tx]) + block_dict = decouple_assets(b, block) inpipe = Pipe() outpipe = Pipe() @@ -263,7 +275,7 @@ def test_valid_block_voting_with_create_transaction(b, vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) - inpipe.put(block.to_dict()) + inpipe.put(block_dict) vote_pipeline.start() vote_out = outpipe.get() vote_pipeline.terminate() diff --git a/tests/test_block_model.py b/tests/test_block_model.py index 6e559cb2..6e14d293 100644 --- a/tests/test_block_model.py +++ b/tests/test_block_model.py @@ -1,3 +1,4 @@ +import pytest from pytest import raises @@ -153,3 +154,137 @@ class TestBlockModel(object): block = b.create_block([tx, tx]) with raises(DuplicateTransaction): block._validate_block(b) + + def test_decouple_assets(self, b): + from bigchaindb.models import Block, Transaction + + assets = [ + {'msg': '1'}, + {'msg': '2'}, + {'msg': '3'}, + ] + + txs = [] + # create 3 assets + for asset in assets: + tx = Transaction.create([b.me], [([b.me], 1)], asset=asset) + txs.append(tx) + + # create a `TRANSFER` transaction. + # the asset in `TRANSFER` transactions is not extracted + tx = Transaction.transfer(txs[0].to_inputs(), [([b.me], 1)], + asset_id=txs[0].id) + txs.append(tx) + + # create the block + block = Block(txs) + # decouple assets + assets_from_block, block_dict = block.decouple_assets() + + assert len(assets_from_block) == 3 + for i in range(3): + assert assets_from_block[i]['data'] == assets[i] + assert assets_from_block[i]['id'] == txs[i].id + + # check the `TRANSFER` transaction was not changed + assert block.transactions[3].to_dict() == \ + block_dict['block']['transactions'][3] + + def test_couple_assets(self, b): + from bigchaindb.models import Block, Transaction + + assets = [ + {'msg': '1'}, + {'msg': '2'}, + {'msg': '3'}, + ] + + txs = [] + # create 3 assets + for asset in assets: + tx = Transaction.create([b.me], [([b.me], 1)], asset=asset) + txs.append(tx) + + # create a `TRANSFER` transaction. + # the asset in `TRANSFER` transactions is not extracted + tx = Transaction.transfer(txs[0].to_inputs(), [([b.me], 1)], + asset_id=txs[0].id) + txs.append(tx) + + # create the block + block = Block(txs) + # decouple assets + assets_from_block, block_dict = block.decouple_assets() + + # reconstruct the block + block_dict_reconstructed = Block.couple_assets(block_dict, + assets_from_block) + + # check that the reconstructed block is the same as the original block + assert block == Block.from_dict(block_dict_reconstructed) + + def test_get_asset_ids(self, b): + from bigchaindb.models import Block, Transaction + + assets = [ + {'msg': '1'}, + {'msg': '2'}, + {'msg': '3'}, + ] + + txs = [] + # create 3 assets + for asset in assets: + tx = Transaction.create([b.me], [([b.me], 1)], asset=asset) + txs.append(tx) + + # create a `TRANSFER` transaction. + # the asset in `TRANSFER` transactions is not extracted + tx = Transaction.transfer(txs[0].to_inputs(), [([b.me], 1)], + asset_id=txs[0].id) + txs.append(tx) + + # create the block + block = Block(txs) + # decouple assets + assets_from_block, block_dict = block.decouple_assets() + + # get the asset_ids and check that they are the same as the `CREATE` + # transactions + asset_ids = Block.get_asset_ids(block_dict) + assert asset_ids == [tx.id for tx in txs[:-1]] + + @pytest.mark.bdb + def test_from_db(self, b): + from bigchaindb.models import Block, Transaction + + assets = [ + {'msg': '1'}, + {'msg': '2'}, + {'msg': '3'}, + ] + + txs = [] + # create 3 assets + for asset in assets: + tx = Transaction.create([b.me], [([b.me], 1)], asset=asset) + txs.append(tx) + + # create a `TRANSFER` transaction. + # the asset in `TRANSFER` transactions is not extracted + tx = Transaction.transfer(txs[0].to_inputs(), [([b.me], 1)], + asset_id=txs[0].id) + txs.append(tx) + + # create the block + block = Block(txs) + # decouple assets + assets_from_block, block_dict = block.decouple_assets() + + # write the assets and block separately + b.write_assets(assets_from_block) + b.write_block(block) + + # check the reconstructed block is the same as the original block + block_from_db = Block.from_db(b, block_dict) + assert block == block_from_db diff --git a/tests/test_fastquery.py b/tests/test_fastquery.py new file mode 100644 index 00000000..8fb3378c --- /dev/null +++ b/tests/test_fastquery.py @@ -0,0 +1,86 @@ +import pytest + +from bigchaindb.common.transaction import TransactionLink +from bigchaindb.models import Block, Transaction + +pytestmark = pytest.mark.bdb + + +@pytest.fixture +def blockdata(b, user_pk, user2_pk): + txs = [Transaction.create([user_pk], [([user2_pk], 1)]), + Transaction.create([user2_pk], [([user_pk], 1)]), + Transaction.create([user_pk], [([user_pk], 1), ([user2_pk], 1)])] + blocks = [] + for i in range(3): + block = Block([txs[i]]) + b.write_block(block) + blocks.append(block.to_dict()) + b.write_vote(b.vote(blocks[1]['id'], '', True)) + b.write_vote(b.vote(blocks[2]['id'], '', False)) + return blocks, [b['id'] for b in blocks] + + +def test_filter_valid_block_ids_with_undecided(b, blockdata): + blocks, block_ids = blockdata + valid_ids = b.fastquery.filter_valid_block_ids(block_ids, include_undecided=True) + assert set(valid_ids) == {blocks[0]['id'], blocks[1]['id']} + + +def test_filter_valid_block_ids_only_valid(b, blockdata): + blocks, block_ids = blockdata + valid_ids = b.fastquery.filter_valid_block_ids(block_ids) + assert set(valid_ids) == {blocks[1]['id']} + + +def test_filter_valid_items(b, blockdata): + blocks, _ = blockdata + assert (b.fastquery.filter_valid_items(blocks, block_id_key=lambda b: b['id']) + == [blocks[0], blocks[1]]) + + +def test_get_outputs_by_public_key(b, user_pk, user2_pk, blockdata): + blocks, _ = blockdata + assert b.fastquery.get_outputs_by_public_key(user_pk) == [ + TransactionLink(blocks[1]['block']['transactions'][0]['id'], 0) + ] + assert b.fastquery.get_outputs_by_public_key(user2_pk) == [ + TransactionLink(blocks[0]['block']['transactions'][0]['id'], 0) + ] + + +def test_filter_spent_outputs(b, user_pk): + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 3) + + # There are 3 inputs + inputs = tx1.to_inputs() + + # Each spent individually + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx3 = Transaction.transfer([inputs[1]], out, tx1.id) + tx4 = Transaction.transfer([inputs[2]], out, tx1.id) + + # The CREATE and first TRANSFER are valid. tx2 produces a new unspent. + for tx in [tx1, tx2]: + block = Block([tx]) + b.write_block(block) + b.write_vote(b.vote(block.id, '', True)) + + # The second TRANSFER is invalid. inputs[1] remains unspent. + block = Block([tx3]) + b.write_block(block) + b.write_vote(b.vote(block.id, '', False)) + + # The third TRANSFER is undecided. It procuces a new unspent. + block = Block([tx4]) + b.write_block(block) + + outputs = b.fastquery.get_outputs_by_public_key(user_pk) + unspents = b.fastquery.filter_spent_outputs(outputs) + + assert set(unspents) == { + inputs[1].fulfills, + tx2.to_inputs()[0].fulfills, + tx4.to_inputs()[0].fulfills + } diff --git a/tests/utils.py b/tests/utils.py index 53ffd86b..9af72a5d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,6 +32,7 @@ def flush_rethink_db(connection, dbname): connection.run(r.db(dbname).table('bigchain').delete()) connection.run(r.db(dbname).table('backlog').delete()) connection.run(r.db(dbname).table('votes').delete()) + connection.run(r.db(dbname).table('assets').delete()) except r.ReqlOpFailedError: pass @@ -41,6 +42,7 @@ def flush_mongo_db(connection, dbname): connection.conn[dbname].bigchain.delete_many({}) connection.conn[dbname].backlog.delete_many({}) connection.conn[dbname].votes.delete_many({}) + connection.conn[dbname].assets.delete_many({}) @singledispatch