diff --git a/.ci/travis-after-success.sh b/.ci/travis-after-success.sh index 2fc659e6..bb00d7a0 100755 --- a/.ci/travis-after-success.sh +++ b/.ci/travis-after-success.sh @@ -3,5 +3,5 @@ set -e -x if [[ -z ${TOXENV} ]]; then - codecov + codecov -v fi diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh index 3757e908..28111c80 100755 --- a/.ci/travis-before-install.sh +++ b/.ci/travis-before-install.sh @@ -1,6 +1,11 @@ #!/bin/bash -apt-get update -qq -wget https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz -O /tmp/v1.1.0.tar.gz -tar -xvf /tmp/v1.1.0.tar.gz -cp -r $PWD/rapidjson-1.1.0/include/rapidjson /usr/include/ +if [[ -z ${TOXENV} ]]; then + sudo apt-get update + sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + + sudo rm /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + chmod +x docker-compose + sudo mv docker-compose /usr/local/bin +fi diff --git a/.ci/travis-before-script.sh b/.ci/travis-before-script.sh index 22a275b1..b3115b3f 100755 --- a/.ci/travis-before-script.sh +++ b/.ci/travis-before-script.sh @@ -2,27 +2,6 @@ set -e -x -if [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then - # Connect to MongoDB on port 27017 via a normal, unsecure connection if - # BIGCHAINDB_DATABASE_SSL is unset. - # It is unset in this case in .travis.yml. - docker pull mongo:3.4 - docker run -d --publish=27017:27017 --name mdb-without-ssl mongo:3.4 # --replSet=bigchain-rs -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then - # Connect to MongoDB on port 27017 via TLS/SSL connection if - # BIGCHAINDB_DATABASE_SSL is set. - # It is set to 'true' here in .travis.yml. Dummy certificates for testing - # are stored under bigchaindb/tests/backend/mongodb-ssl/certs/ directory. - docker pull mongo:3.4 - docker run -d \ - --name mdb-with-ssl \ - --publish=27017:27017 \ - --volume=${TRAVIS_BUILD_DIR}/tests/backend/mongodb-ssl/certs:/certs \ - mongo:3.4 \ - --sslMode=requireSSL \ - --sslCAFile=/certs/ca-chain.cert.pem \ - --sslCRLFile=/certs/crl.pem \ - --sslPEMKeyFile=/certs/local-mongo.pem +if [[ -z ${TOXENV} ]]; then + docker-compose -f docker-compose.travis.yml up -d bdb fi diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 097f81dc..4af4908d 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -7,6 +7,6 @@ pip install --upgrade pip if [[ -n ${TOXENV} ]]; then pip install --upgrade tox else - pip install .[test] + docker-compose -f docker-compose.travis.yml build --no-cache pip install --upgrade codecov fi diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh index b33c40cc..72535932 100755 --- a/.ci/travis_script.sh +++ b/.ci/travis_script.sh @@ -4,16 +4,6 @@ set -e -x if [[ -n ${TOXENV} ]]; then tox -e ${TOXENV} -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then - # Run the full suite of tests for MongoDB over an unsecure connection - pytest -sv --database-backend=localmongodb --cov=bigchaindb -m tendermint -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \ - "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then - # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'. - pytest -sv --database-backend=localmongodb-ssl --cov=bigchaindb -m bdb_ssl else - # Run the full suite of tests for RethinkDB (the default backend when testing) - pytest -sv -m "serial" - pytest -sv --cov=bigchaindb -m "not serial" + docker-compose -f docker-compose.travis.yml run --rm --no-deps bdb pytest -v --cov=bigchaindb fi diff --git a/.gitignore b/.gitignore index 5397ca64..86fa44c1 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ htmlcov/ .coverage .coverage.* .cache +.pytest_cache/ nosetests.xml coverage.xml *.cover diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..c8b8b04c --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,6 @@ +build: + image: latest + +python: + version: 3.6 + pip_install: true diff --git a/.travis.yml b/.travis.yml index d1b281dc..8211e652 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,12 @@ python: - 3.6 env: - - TOXENV=flake8 - - TOXENV=docsroot - - TOXENV=docsserver + global: + - DOCKER_COMPOSE_VERSION=1.19.0 + matrix: + - TOXENV=flake8 + - TOXENV=docsroot + - TOXENV=docsserver matrix: fast_finish: true diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 34c981b7..673fe57f 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -1,6 +1,5 @@ import copy import logging -import os from bigchaindb.log.configs import SUBSCRIBER_LOGGING_CONFIG as log_config @@ -10,9 +9,9 @@ from bigchaindb.log.configs import SUBSCRIBER_LOGGING_CONFIG as log_config _base_database_rethinkdb = { - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), + 'host': 'localhost', + 'port': 28015, + 'name': 'bigchain', } # The following variable is used by `bigchaindb configure` to @@ -27,53 +26,53 @@ _database_keys_map = { } _base_database_localmongodb = { - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET'), - 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), - 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') + 'host': 'localhost', + 'port': 27017, + 'name': 'bigchain', + 'replicaset': None, + 'login': None, + 'password': None, } _base_database_mongodb = { - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), - 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') + 'host': 'localhost', + 'port': 27017, + 'name': 'bigchain', + 'replicaset': 'bigchain-rs', + 'login': None, + 'password': None, } _database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + 'backend': 'rethinkdb', 'connection_timeout': 5000, 'max_tries': 3, } _database_rethinkdb.update(_base_database_rethinkdb) _database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'backend': 'mongodb', 'connection_timeout': 5000, 'max_tries': 3, - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), - 'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT'), - 'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE'), - 'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE'), - 'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE'), - 'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE') + 'ssl': False, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None, } _database_mongodb.update(_base_database_mongodb) _database_localmongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'localmongodb'), + 'backend': 'localmongodb', 'connection_timeout': 5000, 'max_tries': 3, - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), - 'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT'), - 'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE'), - 'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE'), - 'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE'), - 'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE') + 'ssl': False, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None, } _database_localmongodb.update(_base_database_localmongodb) @@ -87,22 +86,21 @@ config = { 'server': { # Note: this section supports all the Gunicorn settings: # - http://docs.gunicorn.org/en/stable/settings.html - 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', + 'bind': 'localhost:9984', 'loglevel': logging.getLevelName( log_config['handlers']['console']['level']).lower(), 'workers': None, # if none, the value will be cpu_count * 2 + 1 }, 'wsserver': { - 'scheme': os.environ.get('BIGCHAINDB_WSSERVER_SCHEME') or 'ws', - 'host': os.environ.get('BIGCHAINDB_WSSERVER_HOST') or 'localhost', - 'port': int(os.environ.get('BIGCHAINDB_WSSERVER_PORT', 9985)), - 'advertised_scheme': os.environ.get('BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME') or 'ws', - 'advertised_host': os.environ.get('BIGCHAINDB_WSSERVER_ADVERTISED_HOST') or 'localhost', - 'advertised_port': int(os.environ.get('BIGCHAINDB_WSSERVER_ADVERTISED_PORT', 9985)), + 'scheme': 'ws', + 'host': 'localhost', + 'port': 9985, + 'advertised_scheme': 'ws', + 'advertised_host': 'localhost', + 'advertised_port': 9985, }, - 'database': _database_map[ - os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') - ], + # FIXME: hardcoding to localmongodb for now + 'database': _database_map['localmongodb'], 'keypair': { 'public': None, 'private': None, @@ -123,9 +121,6 @@ config = { 'granular_levels': {}, 'port': log_config['root']['port'] }, - 'graphite': { - 'host': os.environ.get('BIGCHAINDB_GRAPHITE_HOST', 'localhost'), - }, } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index e167f99e..3a21c802 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -24,11 +24,8 @@ def store_transaction(conn, signed_transaction): @register_query(LocalMongoDBConnection) def store_transactions(conn, signed_transactions): - try: - return conn.run(conn.collection('transactions') - .insert_many(signed_transactions)) - except DuplicateKeyError: - pass + return conn.run(conn.collection('transactions') + .insert_many(signed_transactions)) @register_query(LocalMongoDBConnection) @@ -54,12 +51,9 @@ def get_transactions(conn, transaction_ids): @register_query(LocalMongoDBConnection) def store_metadatas(conn, metadata): - try: - return conn.run( - conn.collection('metadata') - .insert_many(metadata, ordered=False)) - except DuplicateKeyError: - pass + return conn.run( + conn.collection('metadata') + .insert_many(metadata, ordered=False)) @register_query(LocalMongoDBConnection) @@ -82,12 +76,9 @@ def store_asset(conn, asset): @register_query(LocalMongoDBConnection) def store_assets(conn, assets): - try: - return conn.run( - conn.collection('assets') - .insert_many(assets, ordered=False)) - except DuplicateKeyError: - pass + return conn.run( + conn.collection('assets') + .insert_many(assets, ordered=False)) @register_query(LocalMongoDBConnection) @@ -199,3 +190,70 @@ def get_block_with_transaction(conn, txid): conn.collection('blocks') .find({'transactions': txid}, projection={'_id': False, 'height': True})) + + +@register_query(LocalMongoDBConnection) +def delete_zombie_transactions(conn): + txns = conn.run(conn.collection('transactions').find({})) + for txn in txns: + txn_id = txn['id'] + block = list(get_block_with_transaction(conn, txn_id)) + if len(block) == 0: + delete_transaction(conn, txn_id) + + +def delete_transaction(conn, txn_id): + conn.run( + conn.collection('transactions').delete_one({'id': txn_id})) + conn.run( + conn.collection('assets').delete_one({'id': txn_id})) + conn.run( + conn.collection('metadata').delete_one({'id': txn_id})) + + +@register_query(LocalMongoDBConnection) +def delete_latest_block(conn): + block = get_latest_block(conn) + txn_ids = block['transactions'] + delete_transactions(conn, txn_ids) + conn.run(conn.collection('blocks').delete_one({'height': block['height']})) + + +@register_query(LocalMongoDBConnection) +def delete_transactions(conn, txn_ids): + conn.run(conn.collection('assets').delete_many({'id': {'$in': txn_ids}})) + conn.run(conn.collection('metadata').delete_many({'id': {'$in': txn_ids}})) + conn.run(conn.collection('transactions').delete_many({'id': {'$in': txn_ids}})) + + +@register_query(LocalMongoDBConnection) +def store_unspent_outputs(conn, *unspent_outputs): + try: + return conn.run( + conn.collection('utxos') + .insert_many(unspent_outputs, ordered=False)) + except DuplicateKeyError: + # TODO log warning at least + pass + + +@register_query(LocalMongoDBConnection) +def delete_unspent_outputs(conn, *unspent_outputs): + cursor = conn.run( + conn.collection('utxos').remove( + {'$or': [ + {'$and': [ + {'transaction_id': unspent_output['transaction_id']}, + {'output_index': unspent_output['output_index']} + ]} + for unspent_output in unspent_outputs + ]} + )) + return cursor + + +@register_query(LocalMongoDBConnection) +def get_unspent_outputs(conn, *, query=None): + if query is None: + query = {} + return conn.run(conn.collection('utxos').find(query)) diff --git a/bigchaindb/backend/localmongodb/schema.py b/bigchaindb/backend/localmongodb/schema.py index 2d916856..5b152ee8 100644 --- a/bigchaindb/backend/localmongodb/schema.py +++ b/bigchaindb/backend/localmongodb/schema.py @@ -27,7 +27,7 @@ def create_database(conn, dbname): @register_schema(LocalMongoDBConnection) def create_tables(conn, dbname): - for table_name in ['transactions', 'assets', 'blocks', 'metadata']: + for table_name in ['transactions', 'utxos', 'assets', 'blocks', 'metadata']: logger.info('Create `%s` table.', table_name) # create the table # TODO: read and write concerns can be declared here @@ -40,6 +40,7 @@ def create_indexes(conn, dbname): create_assets_secondary_index(conn, dbname) create_blocks_secondary_index(conn, dbname) create_metadata_secondary_index(conn, dbname) + create_utxos_secondary_index(conn, dbname) @register_schema(LocalMongoDBConnection) @@ -99,3 +100,13 @@ def create_metadata_secondary_index(conn, dbname): # full text search index conn.conn[dbname]['metadata'].create_index([('$**', TEXT)], name='text') + + +def create_utxos_secondary_index(conn, dbname): + logger.info('Create `utxos` secondary index.') + + conn.conn[dbname]['utxos'].create_index( + [('transaction_id', ASCENDING), ('output_index', ASCENDING)], + name='utxo', + unique=True, + ) diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 96a27888..1e59b573 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -547,3 +547,62 @@ def store_block(conn, block): """ raise NotImplementedError + + +@singledispatch +def delete_zombie_transactions(conn): + """Delete transactions not included in any block""" + + raise NotImplementedError + + +@singledispatch +def store_unspent_outputs(connection, unspent_outputs): + """Store unspent outputs in ``utxo_set`` table.""" + + raise NotImplementedError + + +@singledispatch +def delete_latest_block(conn): + """Delete the latest block along with its transactions""" + + raise NotImplementedError + + +@singledispatch +def delete_unspent_outputs(connection, unspent_outputs): + """Delete unspent outputs in ``utxo_set`` table.""" + + raise NotImplementedError + + +@singledispatch +def delete_transactions(conn, txn_ids): + """Delete transactions from database + + Args: + txn_ids (list): list of transaction ids + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def get_unspent_outputs(connection, *, query=None): + """Retrieves unspent outputs. + + Args: + query (dict): An optional parameter to filter the result set. + Defaults to ``None``, which means that all UTXO records + will be returned. + + Returns: + Generator yielding unspent outputs (UTXO set) according to the + given query. + """ + + raise NotImplementedError diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index ac81591b..a866fcaa 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -15,8 +15,10 @@ from bigchaindb.common.exceptions import (StartupError, KeypairNotFoundException, DatabaseDoesNotExist) import bigchaindb +from bigchaindb.tendermint.core import BigchainDB from bigchaindb import backend from bigchaindb.backend import schema +from bigchaindb.backend import query from bigchaindb.backend.admin import (set_replicas, set_shards, add_replicas, remove_replicas) from bigchaindb.backend.exceptions import OperationError @@ -154,6 +156,19 @@ def run_init(args): print('If you wish to re-initialize it, first drop it.', file=sys.stderr) +def run_recover(b): + query.delete_zombie_transactions(b.connection) + + tendermint_height = b.get_latest_block_height_from_tendermint() + block = b.get_latest_block() + + if block: + while block['height'] > tendermint_height: + logger.info('BigchainDB is ahead of tendermint, removing block %s', block['height']) + query.delete_latest_block(b.connection) + block = b.get_latest_block() + + @configure_bigchaindb def run_drop(args): """Drop the database""" @@ -178,6 +193,8 @@ def run_start(args): """Start the processes to run the node""" logger.info('BigchainDB Version %s', bigchaindb.__version__) + run_recover(BigchainDB()) + if args.allow_temp_keypair: if not (bigchaindb.config['keypair']['private'] or bigchaindb.config['keypair']['public']): @@ -271,7 +288,10 @@ def create_parser(): help='Prepare the config file ' 'and create the node keypair') config_parser.add_argument('backend', - choices=['rethinkdb', 'mongodb', 'localmongodb'], + choices=['localmongodb'], + default='localmongodb', + const='localmongodb', + nargs='?', help='The backend to use. It can be either ' 'rethinkdb or mongodb.') diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md index 76ff7f28..d43d1985 100644 --- a/bigchaindb/common/schema/README.md +++ b/bigchaindb/common/schema/README.md @@ -4,22 +4,24 @@ This directory contains the schemas for the different JSON documents BigchainDB The aim is to provide: -- a strict definition of the data structures used in BigchainDB -- a language independent tool to validate the structure of incoming/outcoming - data (there are several ready to use +- a strict definition of the data structures used in BigchainDB, +- a language-independent tool to validate the structure of incoming/outcoming + data. (There are several ready to use [implementations](http://json-schema.org/implementations.html) written in - different languages) + different languages.) ## Sources -The file defining the JSON Schema for votes (`vote.yaml`) is BigchainDB-specific. - The files defining the JSON Schema for transactions (`transaction_*.yaml`) -are copied from the [IPDB Protocol](https://github.com/ipdb/ipdb-protocol). -If you want to add a new version, you must add it to the IPDB Protocol first. -(You can't change existing versions. Those were used to validate old transactions +are based on the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). +If you want to add a new transaction version, +you must add it to the IPDB Transaction Spec first. +(You can't change the JSON Schema files for old versions. +Those were used to validate old transactions and are needed to re-check those transactions.) +The file defining the JSON Schema for votes (`vote.yaml`) is BigchainDB-specific. + ## Learn about JSON Schema A good resource is [Understanding JSON Schema](http://spacetelescope.github.io/understanding-json-schema/index.html). diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 0f4990dd..e64e5640 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -22,7 +22,7 @@ def _load_schema(name): return path, (schema, fast_schema) -TX_SCHEMA_VERSION = 'v1.0' +TX_SCHEMA_VERSION = 'v2.0' TX_SCHEMA_PATH, TX_SCHEMA_COMMON = _load_schema('transaction_' + TX_SCHEMA_VERSION) diff --git a/bigchaindb/common/schema/transaction_create_v2.0.yaml b/bigchaindb/common/schema/transaction_create_v2.0.yaml new file mode 100644 index 00000000..3d393347 --- /dev/null +++ b/bigchaindb/common/schema/transaction_create_v2.0.yaml @@ -0,0 +1,30 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - CREATE/GENESIS specific constraints +required: +- asset +- inputs +properties: + asset: + additionalProperties: false + properties: + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + required: + - data + inputs: + type: array + title: "Transaction inputs" + maxItems: 1 + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "null" diff --git a/bigchaindb/common/schema/transaction_transfer_v2.0.yaml b/bigchaindb/common/schema/transaction_transfer_v2.0.yaml new file mode 100644 index 00000000..538ec5e6 --- /dev/null +++ b/bigchaindb/common/schema/transaction_transfer_v2.0.yaml @@ -0,0 +1,29 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - TRANSFER specific properties +required: +- asset +properties: + asset: + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + required: + - id + inputs: + type: array + title: "Transaction inputs" + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "object" +definitions: + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string diff --git a/bigchaindb/common/schema/transaction_v2.0.yaml b/bigchaindb/common/schema/transaction_v2.0.yaml new file mode 100644 index 00000000..16cafd33 --- /dev/null +++ b/bigchaindb/common/schema/transaction_v2.0.yaml @@ -0,0 +1,163 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +additionalProperties: false +title: Transaction Schema +required: +- id +- inputs +- outputs +- operation +- metadata +- asset +- version +properties: + id: + anyOf: + - "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" + inputs: + type: array + title: "Transaction inputs" + items: + "$ref": "#/definitions/input" + outputs: + type: array + items: + "$ref": "#/definitions/output" + metadata: + "$ref": "#/definitions/metadata" + version: + type: string + pattern: "^2\\.0$" +definitions: + offset: + type: integer + minimum: 0 + base58: + pattern: "[1-9a-zA-Z^OIl]{43,44}" + type: string + public_keys: + anyOf: + - type: array + items: + "$ref": "#/definitions/base58" + - type: 'null' + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string + uuid4: + pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" + type: string + operation: + type: string + enum: + - CREATE + - TRANSFER + - GENESIS + asset: + type: object + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + output: + type: object + additionalProperties: false + required: + - amount + - condition + - public_keys + properties: + amount: + type: string + pattern: "^[0-9]{1,20}$" + condition: + type: object + additionalProperties: false + required: + - details + - uri + properties: + details: + "$ref": "#/definitions/condition_details" + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" + public_keys: + "$ref": "#/definitions/public_keys" + input: + type: "object" + additionalProperties: false + required: + - owners_before + - fulfillment + properties: + owners_before: + "$ref": "#/definitions/public_keys" + fulfillment: + anyOf: + - type: string + pattern: "^[a-zA-Z0-9_-]*$" + - "$ref": "#/definitions/condition_details" + fulfills: + anyOf: + - type: 'object' + additionalProperties: false + required: + - output_index + - transaction_id + properties: + output_index: + "$ref": "#/definitions/offset" + transaction_id: + "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + metadata: + anyOf: + - type: object + additionalProperties: true + minProperties: 1 + - type: 'null' + condition_details: + anyOf: + - type: object + additionalProperties: false + required: + - type + - public_key + properties: + type: + type: string + pattern: "^ed25519-sha-256$" + public_key: + "$ref": "#/definitions/base58" + - type: object + additionalProperties: false + required: + - type + - threshold + - subconditions + properties: + type: + type: "string" + pattern: "^threshold-sha-256$" + threshold: + type: integer + minimum: 1 + maximum: 100 + subconditions: + type: array + items: + "$ref": "#/definitions/condition_details" diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index e4738ca1..13af253c 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1,7 +1,12 @@ """Transaction related models to parse and construct transaction payloads. +Attributes: + UnspentOutput (namedtuple): Object holding the information + representing an unspent output. + """ +from collections import namedtuple from copy import deepcopy from functools import reduce @@ -19,6 +24,17 @@ from bigchaindb.common.exceptions import (KeypairMismatchException, from bigchaindb.common.utils import serialize +UnspentOutput = namedtuple( + 'UnspentOutput', ( + 'transaction_id', + 'output_index', + 'amount', + 'asset_id', + 'condition_uri', + ) +) + + class Input(object): """A Input is used to spend assets locked by an Output. @@ -475,7 +491,7 @@ class Transaction(object): TRANSFER = 'TRANSFER' GENESIS = 'GENESIS' ALLOWED_OPERATIONS = (CREATE, TRANSFER, GENESIS) - VERSION = '1.0' + VERSION = '2.0' def __init__(self, operation, asset, inputs=None, outputs=None, metadata=None, version=None, hash_id=None): @@ -532,6 +548,35 @@ class Transaction(object): self.metadata = metadata self._id = hash_id + @property + def unspent_outputs(self): + """UnspentOutput: The outputs of this transaction, in a data + structure containing relevant information for storing them in + a UTXO set, and performing validation. + """ + if self.operation == Transaction.CREATE: + self._asset_id = self._id + elif self.operation == Transaction.TRANSFER: + self._asset_id = self.asset['id'] + return (UnspentOutput( + transaction_id=self._id, + output_index=output_index, + amount=output.amount, + asset_id=self._asset_id, + condition_uri=output.fulfillment.condition_uri, + ) for output_index, output in enumerate(self.outputs)) + + @property + def spent_outputs(self): + """tuple of :obj:`dict`: Inputs of this transaction. Each input + is represented as a dictionary containing a transaction id and + output index. + """ + return ( + input_.fulfills.to_dict() + for input_ in self.inputs if input_.fulfills + ) + @property def serialized(self): return Transaction._to_str(self.to_dict()) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 4fb84722..dab88045 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,5 +1,4 @@ import random -import statsd from time import time from bigchaindb import exceptions as core_exceptions @@ -72,8 +71,6 @@ class Bigchain(object): if not self.me or not self.me_private: raise exceptions.KeypairNotFoundException() - self.statsd = statsd.StatsClient(bigchaindb.config['graphite']['host']) - federation = property(lambda self: set(self.nodes_except_me + [self.me])) """ Set of federation member public keys """ diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index 4d482255..8c90313a 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -126,8 +126,6 @@ class BlockPipeline: logger.info('Write new block %s with %s transactions', block.id, len(block.transactions)) self.bigchain.write_block(block) - self.bigchain.statsd.incr('pipelines.block.throughput', - len(block.transactions)) return block def delete_tx(self, block): diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index f6959da0..fe19da58 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -149,7 +149,6 @@ class Vote: logger.info("Voting '%s' for block %s", validity, vote['vote']['voting_for_block']) self.bigchain.write_vote(vote) - self.bigchain.statsd.incr('pipelines.vote.throughput', num_tx) return vote diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index 765fb190..2191924b 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -11,8 +11,8 @@ from bigchaindb.events import EventTypes, Event from bigchaindb.tendermint.utils import decode_transaction_base64 -HOST = getenv('TENDERMINT_HOST', 'localhost') -PORT = int(getenv('TENDERMINT_PORT', 46657)) +HOST = getenv('BIGCHAINDB_TENDERMINT_HOST', 'localhost') +PORT = int(getenv('BIGCHAINDB_TENDERMINT_PORT', 46657)) URL = f'ws://{HOST}:{PORT}/websocket' logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ def connect_and_recv(event_queue): logger.info('Connected to tendermint ws server') - stream_id = "bigchaindb_stream_{}".format(gen_timestamp()) + stream_id = 'bigchaindb_stream_{}'.format(gen_timestamp()) yield from subscribe_events(ws, stream_id) while True: @@ -58,10 +58,10 @@ def process_event(event_queue, event, stream_id): @asyncio.coroutine def subscribe_events(ws, stream_id): payload = { - "method": "subscribe", - "jsonrpc": "2.0", - "params": ["NewBlock"], - "id": stream_id + 'method': 'subscribe', + 'jsonrpc': '2.0', + 'params': ['NewBlock'], + 'id': stream_id } yield from ws.send_str(json.dumps(payload)) diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 37788b51..88f65ddd 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -17,9 +17,12 @@ from bigchaindb import exceptions as core_exceptions logger = logging.getLogger(__name__) -TENDERMINT_HOST = getenv('TENDERMINT_HOST', 'localhost') -TENDERMINT_PORT = getenv('TENDERMINT_PORT', '46657') -ENDPOINT = 'http://{}:{}/'.format(TENDERMINT_HOST, TENDERMINT_PORT) +BIGCHAINDB_TENDERMINT_HOST = getenv('BIGCHAINDB_TENDERMINT_HOST', + 'localhost') +BIGCHAINDB_TENDERMINT_PORT = getenv('BIGCHAINDB_TENDERMINT_PORT', + '46657') +ENDPOINT = 'http://{}:{}/'.format(BIGCHAINDB_TENDERMINT_HOST, + BIGCHAINDB_TENDERMINT_PORT) MODE_LIST = ('broadcast_tx_async', 'broadcast_tx_sync', 'broadcast_tx_commit') @@ -47,6 +50,10 @@ class BigchainDB(Bigchain): """Submit a valid transaction to the mempool.""" self.post_transaction(transaction, mode) + def get_latest_block_height_from_tendermint(self): + r = requests.get(ENDPOINT + 'status') + return r.json()['result']['latest_block_height'] + def store_transaction(self, transaction): """Store a valid transaction to the transactions collection.""" diff --git a/bigchaindb/web/routes.py b/bigchaindb/web/routes.py index b5fd8eb3..5b6185ba 100644 --- a/bigchaindb/web/routes.py +++ b/bigchaindb/web/routes.py @@ -5,7 +5,6 @@ from bigchaindb.web.views import ( metadata, blocks, info, - statuses, transactions as tx, outputs, votes, @@ -31,7 +30,6 @@ ROUTES_API_V1 = [ r('metadata/', metadata.MetadataApi), r('blocks/', blocks.BlockApi), r('blocks/', blocks.BlockListApi), - r('statuses/', statuses.StatusApi), r('transactions/', tx.TransactionApi), r('transactions', tx.TransactionListApi), r('outputs/', outputs.OutputListApi), diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index d6162a34..205f4930 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -46,7 +46,6 @@ def get_api_v1_info(api_prefix): return { 'docs': ''.join(docs_url), 'transactions': '{}transactions/'.format(api_prefix), - 'statuses': '{}statuses/'.format(api_prefix), 'assets': '{}assets/'.format(api_prefix), 'outputs': '{}outputs/'.format(api_prefix), 'streams': websocket_root, diff --git a/bigchaindb/web/views/statuses.py b/bigchaindb/web/views/statuses.py deleted file mode 100644 index 0044c334..00000000 --- a/bigchaindb/web/views/statuses.py +++ /dev/null @@ -1,45 +0,0 @@ -"""This module provides the blueprint for the statuses API endpoints. - -For more information please refer to the documentation: http://bigchaindb.com/http-api -""" -from flask import current_app -from flask_restful import Resource, reqparse - -from bigchaindb.web.views.base import make_error - - -class StatusApi(Resource): - def get(self): - """API endpoint to get details about the status of a transaction or a block. - - Return: - A ``dict`` in the format ``{'status': }``, where - ```` is one of "valid", "invalid", "undecided", "backlog". - """ - parser = reqparse.RequestParser() - parser.add_argument('transaction_id', type=str) - parser.add_argument('block_id', type=str) - - args = parser.parse_args(strict=True) - tx_id = args['transaction_id'] - block_id = args['block_id'] - - # logical xor - exactly one query argument required - if bool(tx_id) == bool(block_id): - return make_error(400, 'Provide exactly one query parameter. Choices are: block_id, transaction_id') - - pool = current_app.config['bigchain_pool'] - status = None - - with pool() as bigchain: - if tx_id: - status = bigchain.get_status(tx_id) - elif block_id: - _, status = bigchain.get_block(block_id=block_id, include_status=True) - - if not status: - return make_error(404) - - return { - 'status': status - } diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index c746f2ad..783d020a 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -82,7 +82,6 @@ class TransactionListApi(Resource): ) with pool() as bigchain: - bigchain.statsd.incr('web.tx.post') try: bigchain.validate_transaction(tx_obj) except ValidationError as e: @@ -96,13 +95,4 @@ class TransactionListApi(Resource): response = jsonify(tx) response.status_code = 202 - # NOTE: According to W3C, sending a relative URI is not allowed in the - # Location Header: - # - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html - # - # Flask is autocorrecting relative URIs. With the following command, - # we're able to prevent this. - response.autocorrect_location_header = False - status_monitor = '../statuses?transaction_id={}'.format(tx_obj.id) - response.headers['Location'] = status_monitor return response diff --git a/compose/bigchaindb-server/Dockerfile b/compose/bigchaindb-server/Dockerfile index 4abd8af8..56c63e22 100644 --- a/compose/bigchaindb-server/Dockerfile +++ b/compose/bigchaindb-server/Dockerfile @@ -22,7 +22,7 @@ ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws ENV BIGCHAINDB_START_TENDERMINT 0 -ENV TENDERMINT_PORT 46657 +ENV BIGCHAINDB_TENDERMINT_PORT 46657 RUN mkdir -p /usr/src/app diff --git a/compose/travis/Dockerfile b/compose/travis/Dockerfile new file mode 100644 index 00000000..cb37fa07 --- /dev/null +++ b/compose/travis/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.6 +LABEL maintainer "dev@bigchaindb.com" + +RUN apt-get update \ + && pip install -U pip \ + && apt-get autoremove \ + && apt-get clean + +ARG backend + +ENV PYTHONUNBUFFERED 0 + +ENV BIGCHAINDB_DATABASE_PORT 27017 +ENV BIGCHAINDB_DATABASE_BACKEND $backend +ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 +ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_SCHEME ws + +ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws + +ENV BIGCHAINDB_START_TENDERMINT 0 +ENV BIGCHAINDB_TENDERMINT_PORT 46657 + +RUN mkdir -p /usr/src/app +COPY . /usr/src/app/ +WORKDIR /usr/src/app +RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf +RUN pip install --no-cache-dir .[test] +RUN bigchaindb -y configure "$backend" diff --git a/docker-compose.benchmark.yml b/docker-compose.benchmark.yml index c7319040..fa71875f 100644 --- a/docker-compose.benchmark.yml +++ b/docker-compose.benchmark.yml @@ -22,16 +22,6 @@ services: BIGCHAINDB_DATABASE_HOST: mdb BIGCHAINDB_DATABASE_PORT: 27017 BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 - BIGCHAINDB_GRAPHITE_HOST: graphite ports: - "9984" command: bigchaindb start - - graphite: - image: hopsoft/graphite-statsd - ports: - - "2003-2004" - - "2023-2024" - - "8125/udp" - - "8126" - - "80" diff --git a/docker-compose.network.yml b/docker-compose.network.yml index 38a4b5c4..eb2cdfa6 100644 --- a/docker-compose.network.yml +++ b/docker-compose.network.yml @@ -27,7 +27,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-one - TENDERMINT_HOST: tendermint-one + BIGCHAINDB_TENDERMINT_HOST: tendermint-one ports: - "9984" command: bigchaindb -l DEBUG start @@ -64,7 +64,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-two - TENDERMINT_HOST: tendermint-two + BIGCHAINDB_TENDERMINT_HOST: tendermint-two ports: - "9984" command: bigchaindb -l DEBUG start @@ -101,7 +101,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-three - TENDERMINT_HOST: tendermint-three + BIGCHAINDB_TENDERMINT_HOST: tendermint-three ports: - "9984" command: bigchaindb -l DEBUG start @@ -138,7 +138,7 @@ services: environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb-four - TENDERMINT_HOST: tendermint-four + BIGCHAINDB_TENDERMINT_HOST: tendermint-four ports: - "9984" command: bigchaindb -l DEBUG start diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 7c4489aa..d43af4d9 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -18,6 +18,11 @@ services: volumes: - ./bigchaindb:/usr/src/app/bigchaindb - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb @@ -25,17 +30,21 @@ services: BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 BIGCHAINDB_START_TENDERMINT: 0 - TENDERMINT_HOST: tendermint - TENDERMINT_PORT: 46657 + BIGCHAINDB_TENDERMINT_HOST: tendermint + BIGCHAINDB_TENDERMINT_PORT: 46657 ports: - "9984" + - "46658" command: bigchaindb -l DEBUG start tendermint: image: tendermint/tendermint:0.13 volumes: - - ./tmdata:/tendermint + - ./tmdata/config.toml:/tendermint/config.toml entrypoint: '' - command: bash -c "tendermint unsafe_reset_all && tendermint node" + ports: + - "46656" + - "46657" + command: bash -c "tendermint init && tendermint node" curl-client: image: appropriate/curl command: /bin/sh -c "curl http://tendermint:46657/abci_query && curl http://bdb:9984/" diff --git a/docker-compose.travis.yml b/docker-compose.travis.yml new file mode 100644 index 00000000..7707a276 --- /dev/null +++ b/docker-compose.travis.yml @@ -0,0 +1,34 @@ +version: '3' + +services: + mdb: + image: mongo:3.4.3 + command: mongod + bdb: + depends_on: + - mdb + - tendermint + build: + context: . + dockerfile: ./compose/travis/Dockerfile + args: + backend: localmongodb + volumes: + - .:/usr/src/app/ + environment: + BIGCHAINDB_DATABASE_BACKEND: localmongodb + BIGCHAINDB_DATABASE_HOST: mdb + BIGCHAINDB_DATABASE_PORT: 27017 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 + BIGCHAINDB_START_TENDERMINT: 0 + BIGCHAINDB_TENDERMINT_HOST: tendermint + BIGCHAINDB_TENDERMINT_PORT: 46657 + command: bigchaindb start + tendermint: + image: tendermint/tendermint:0.13 + volumes: + - ./tmdata/config.toml:/tendermint/config.toml + - ./tmdata/genesis.json:/tendermint/genesis.json + - ./tmdata/priv_validator.json:/tendermint/priv_validator.json + entrypoint: ["/bin/tendermint", "node", "--proxy_app=dummy"] diff --git a/docs/README.md b/docs/README.md index 220569e2..4071546f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,14 @@ If you want to generate the HTML version of the long-form documentation on your pip install -r requirements.txt ``` +If you're building the *Server* docs (in `docs/server`) then you must also do: +```bash +pip install -e ../../ +``` + +Note: Don't put `-e ../../` in the `requirements.txt` file. That will work locally +but not on ReadTheDocs. + You can then generate the HTML documentation _in that subdirectory_ by doing: ```bash make html diff --git a/docs/root/source/assets.rst b/docs/root/source/assets.rst index a0aa5a31..dd1a241d 100644 --- a/docs/root/source/assets.rst +++ b/docs/root/source/assets.rst @@ -13,4 +13,4 @@ BigchainDB can store data of any kind (within reason), but it's designed to be p .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the note about "owners" in the `IPDB Transaction Spec `_. diff --git a/docs/root/source/decentralized.md b/docs/root/source/decentralized.md index 755a84a2..f38e12a2 100644 --- a/docs/root/source/decentralized.md +++ b/docs/root/source/decentralized.md @@ -4,7 +4,7 @@ Decentralization means that no one owns or controls everything, and there is no Ideally, each node in a BigchainDB cluster is owned and controlled by a different person or organization. Even if the cluster lives within one organization, it's still preferable to have each node controlled by a different person or subdivision. -We use the phrase "BigchainDB consortium" (or just "consortium") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A consortium requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each consortium, but it can be very decentralized (e.g. purely vote-based, where each node gets a vote, and there are no special leadership roles). +We use the phrase "BigchainDB consortium" (or just "consortium") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A consortium requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each consortium, but it can be very decentralized. If sharding is turned on (i.e. if the number of shards is larger than one), then the actual data is decentralized in that no one node stores all the data. diff --git a/docs/root/source/immutable.md b/docs/root/source/immutable.md index 55b61c27..8791c04e 100644 --- a/docs/root/source/immutable.md +++ b/docs/root/source/immutable.md @@ -13,7 +13,7 @@ Blockchain data can achieve immutability in several ways: 1. **External watchdogs.** A consortium may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For a consortium with publicly-readable data, the public can act as an auditor. 1. **Economic incentives.** Some blockchain systems make it very expensive to change old stored data. Examples include proof-of-work and proof-of-stake systems. BigchainDB doesn't use explicit incentives like those. 1. Data can be stored using fancy techniques, such as error-correction codes, to make some kinds of changes easier to undo. -1. **Cryptographic signatures** are often used as a way to check if messages (e.g. transactions, blocks or votes) have been tampered with enroute, and as a way to verify who signed the messages. In BigchainDB, each transaction must be signed (by one or more parties), each block is signed by the node that created it, and each vote is signed by the node that cast it. +1. **Cryptographic signatures** are often used as a way to check if messages (e.g. transactions) have been tampered with enroute, and as a way to verify who signed the messages. In BigchainDB, each transaction must be signed by one or more parties. 1. **Full or partial backups** may be recorded from time to time, possibly on magnetic tape storage, other blockchains, printouts, etc. 1. **Strong security.** Node owners can adopt and enforce strong security policies. 1. **Node diversity.** Diversity makes it so that no one thing (e.g. natural disaster or operating system bug) can compromise enough of the nodes. See [the section on the kinds of node diversity](diversity.html). diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst index 433075c1..50c4674c 100644 --- a/docs/root/source/index.rst +++ b/docs/root/source/index.rst @@ -60,9 +60,6 @@ At a high level, one can communicate with a BigchainDB cluster (set of nodes) us
JavaScript Driver Docs
-
- Command Line Transaction Tool -
Server Docs
diff --git a/docs/root/source/permissions.rst b/docs/root/source/permissions.rst index 02029e10..68d47bc8 100644 --- a/docs/root/source/permissions.rst +++ b/docs/root/source/permissions.rst @@ -15,8 +15,8 @@ To spend/transfer an unspent output, a user (or group of users) must fulfill the - "…three of these four people must sign." - "…either Bob must sign, or both Tom and Sylvia must sign." -For details, see -`the documentation about conditions in the IPDB Transaction Spec `_. +For details, see the section about conditions +in the `IPDB Transaction Spec `_. Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output. diff --git a/docs/root/source/smart-contracts.rst b/docs/root/source/smart-contracts.rst index 143d6dfe..18b8a21f 100644 --- a/docs/root/source/smart-contracts.rst +++ b/docs/root/source/smart-contracts.rst @@ -15,4 +15,4 @@ Crypto-conditions can be quite complex. They can't include loops or recursion an .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the note about "owners" in the `IPDB Transaction Spec `_. diff --git a/docs/root/source/transaction-concepts.md b/docs/root/source/transaction-concepts.md index 02820ac5..93be77e4 100644 --- a/docs/root/source/transaction-concepts.md +++ b/docs/root/source/transaction-concepts.md @@ -30,14 +30,14 @@ Each output also has an associated condition: the condition that must be met BigchainDB supports a variety of conditions, a subset of the [Interledger Protocol (ILP)](https://interledger.org/) crypto-conditions. For details, see -[the documentation about conditions in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-components/conditions.html). +the section about conditions in the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). Each output also has a list of all the public keys associated with the conditions on that output. Loosely speaking, that list might be interpreted as the list of "owners." A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. -See the [note about "owners" in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/ownership.html). +See the note about "owners" in the [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec). A CREATE transaction must be signed by all the owners. (If you're looking for that signature, @@ -87,7 +87,7 @@ things. We documented those things in a post on *The BigchainDB Blog*: ["What is a Valid Transaction in BigchainDB?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598) (Note: That post was about BigchainDB Server v1.0.0.) -The [IPDB Transaction Spec documents the conditions for a transaction to be valid](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-validation.html). +The [IPDB Transaction Spec](https://github.com/ipdb/ipdb-tx-spec) documents the conditions for a transaction to be valid. ## Example Transactions diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 7e683bb6..27aa8205 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -69,67 +69,12 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ HTTP/1.1 202 Accepted -Location: ../statuses?transaction_id=%(txid)s Content-Type: application/json %(tx)s """ -TPLS['get-statuses-tx-request'] = """\ -GET /statuses?transaction_id=%(txid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-statuses-tx-invalid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "invalid" -} -""" - - -TPLS['get-statuses-tx-valid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "valid" -} -""" - - -TPLS['get-statuses-block-request'] = """\ -GET /api/v1/statuses?block_id=%(blockid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-statuses-block-invalid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "invalid" -} -""" - - -TPLS['get-statuses-block-valid-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "valid" -} -""" - - TPLS['get-block-request'] = """\ GET /api/v1/blocks/%(blockid)s HTTP/1.1 Host: example.com @@ -160,21 +105,6 @@ Content-Type: application/json """ -TPLS['get-vote-request'] = """\ -GET /api/v1/votes?block_id=%(blockid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-vote-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -[%(vote)s] -""" - - def main(): """ Main function """ @@ -258,18 +188,6 @@ def main(): ctx['block_list'] = pretty_json(block_list) - # block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) - block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, - voters=[node_public], signature=signature) - ctx['block_transfer'] = pretty_json(block_transfer.to_dict()) - - # vote - vblock = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) - DUMMY_SHA3 = '0123456789abcdef' * 4 - b = Bigchain(public_key=node_public, private_key=node_private) - vote = b.vote(vblock.id, DUMMY_SHA3, True) - ctx['vote'] = pretty_json(vote) - base_path = os.path.join(os.path.dirname(__file__), 'source/http-samples') if not os.path.exists(base_path): diff --git a/docs/server/requirements.txt b/docs/server/requirements.txt index c68268c9..54e2cd4d 100644 --- a/docs/server/requirements.txt +++ b/docs/server/requirements.txt @@ -5,4 +5,3 @@ sphinxcontrib-napoleon>=0.4.4 sphinxcontrib-httpdomain>=1.5.0 pyyaml>=3.12 aafigure>=0.6 -bigchaindb diff --git a/docs/server/source/appendices/cryptography.rst b/docs/server/source/appendices/cryptography.rst index a03e073b..9347fa25 100644 --- a/docs/server/source/appendices/cryptography.rst +++ b/docs/server/source/appendices/cryptography.rst @@ -1,9 +1,8 @@ Cryptography ============ -See `the IPDB Transaction Spec -`_, -especially the pages about: +See the `IPDB Transaction Spec `_, +especially the sections about: - Cryptographic Hashes - Cryptographic Keys & Signatures diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 588f7ecd..db1695b5 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -18,4 +18,3 @@ Appendices licenses run-with-vagrant run-with-ansible - vote-yaml diff --git a/docs/server/source/appendices/json-serialization.rst b/docs/server/source/appendices/json-serialization.rst index fc67080d..e5b7ac15 100644 --- a/docs/server/source/appendices/json-serialization.rst +++ b/docs/server/source/appendices/json-serialization.rst @@ -1,6 +1,5 @@ JSON Serialization ================== -See the page about JSON Serialization & Deserialization -in `the IPDB Transaction Spec -`_. +See the section about JSON Serialization & Deserialization +in the `IPDB Transaction Spec `_. diff --git a/docs/server/source/appendices/vote-yaml.rst b/docs/server/source/appendices/vote-yaml.rst deleted file mode 100644 index 9ddb5541..00000000 --- a/docs/server/source/appendices/vote-yaml.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. _the-vote-schema-file: - -The Vote Schema File -==================== - -BigchainDB checks all :ref:`votes ` -(JSON documents) against a formal schema -defined in a JSON Schema file named vote.yaml. -The contents of that file are copied below. -To understand those contents -(i.e. JSON Schema), check out -`"Understanding JSON Schema" -`_ -by Michael Droettboom or -`json-schema.org `_. - - -vote.yaml ---------- - -.. literalinclude:: ../../../../bigchaindb/common/schema/vote.yaml - :language: yaml \ No newline at end of file diff --git a/docs/server/source/data-models/asset-model.rst b/docs/server/source/data-models/asset-model.rst index 2046422f..1d6bc13e 100644 --- a/docs/server/source/data-models/asset-model.rst +++ b/docs/server/source/data-models/asset-model.rst @@ -1,5 +1,4 @@ The Asset Model =============== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/conditions.rst b/docs/server/source/data-models/conditions.rst index 4eb8898a..4e299b9f 100644 --- a/docs/server/source/data-models/conditions.rst +++ b/docs/server/source/data-models/conditions.rst @@ -1,5 +1,4 @@ Conditions ========== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/index.rst b/docs/server/source/data-models/index.rst index 2827d219..94eb584e 100644 --- a/docs/server/source/data-models/index.rst +++ b/docs/server/source/data-models/index.rst @@ -9,4 +9,3 @@ Data Models inputs-outputs conditions block-model - vote-model diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index f460d216..8f4e8762 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -1,5 +1,4 @@ Inputs and Outputs ================== -See `the IPDB Transaction Spec -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 7f7721cc..e406236e 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -3,22 +3,4 @@ The Transaction Model ===================== -See `the IPDB Transaction Spec -`_. - - -The Transaction Schema ----------------------- - -BigchainDB checks all transactions (JSON documents) -against a formal schema defined -in some `JSON Schema `_ files. -Those files are part of the IPDB Transaction Spec. -Their official source is the ``tx_schema/`` directory -in the `ipdb/ipdb-tx-spec repository on GitHub -`_, -but BigchainDB Server uses copies of those files; -those copies can be found -in the ``bigchaindb/common/schema/`` directory -in the `bigchaindb/bigchaindb repository on GitHub -`_. +See the `IPDB Transaction Spec `_. diff --git a/docs/server/source/data-models/vote-model.rst b/docs/server/source/data-models/vote-model.rst deleted file mode 100644 index 9a0d1689..00000000 --- a/docs/server/source/data-models/vote-model.rst +++ /dev/null @@ -1,123 +0,0 @@ -.. _the-vote-model: - -The Vote Model -============== - -A vote is a JSON object with a particular schema, -as outlined in this page. -A vote must contain the following JSON keys -(also called names or fields): - -.. code-block:: json - - { - "node_pubkey": "", - "vote": { - "voting_for_block": "", - "previous_block": "", - "is_block_valid": "", - "invalid_reason": null, - "timestamp": "" - }, - "signature": "" - } - -.. note:: - - Votes have no ID (or ``"id"``), as far as users are concerned. - The backend database may use one internally, - but it's of no concern to users and it's never reported to them via APIs. - - -The JSON Keys in a Vote ------------------------ - -**node_pubkey** - -The public key of the node which cast this vote. -It's a string. -For more information about public keys, -see the `IPDB Transaction Spec page about cryptographic keys and signatures -`_. - - -**vote.voting_for_block** - -The block ID that this vote is for. -It's a string. -For more information about block IDs, -see the page about :ref:`blocks `. - - -**vote.previous_block** - -The block ID of the block "before" the block that this vote is for, -according to the node which cast this vote. -It's a string. -(It's possible for different nodes to see different block orders.) -For more information about block IDs, -see the page about :ref:`blocks `. - - -**vote.is_block_valid** - -``true`` if the node which cast this vote considered the block in question to be valid, -and ``false`` otherwise. -Note that it's a *boolean* (i.e. ``true`` or ``false``), not a string. - - -**vote.invalid_reason** - -Always ``null``, that is, it's not being used. -It may be used or dropped in a future version. -See `bigchaindb/bigchaindb issue #217 -`_ on GitHub. - - -**vote.timestamp** - -The `Unix time `_ -when the vote was created, according to the node which created it. -It's a string representation of an integer. - - -**signature** - -The cryptographic signature of the inner ``vote`` -by the node that created the vote -(i.e. the node with public key ``node_pubkey``). -To compute that: - -#. Construct an :term:`associative array` ``d`` containing the contents of the inner ``vote`` - (i.e. ``vote.voting_for_block``, ``vote.previous_block``, ``vote.is_block_valid``, - ``vote.invalid_reason``, ``vote.timestamp``, and their values). -#. Compute ``signature = sig_of_aa(d, private_key)``, where ``private_key`` - is the node's private key (i.e. ``node_pubkey`` and ``private_key`` are a key pair). - There's pseudocode for the ``sig_of_aa()`` function - on `the IPDB Transaction Spec page about cryptographic keys and signatures - `_. - - -The Vote Schema ---------------- - -BigchainDB checks all votes (JSON documents) against a formal schema -defined in a :ref:`JSON Schema file named vote.yaml `. - - -An Example Vote ---------------- - -.. code-block:: json - - { - "node_pubkey": "3ZCsVWPAhPTqHx9wZVxp9Se54pcNeeM5mQvnozDWyDR9", - "vote": { - "voting_for_block": "11c3a3fcc9efa4fc4332a0849fc39b58e403ff37794a7d1fdfb9e7703a94a274", - "previous_block": "3dd1441018b782a50607dc4c7f83a0f0a23eb257f4b6a8d99330dfff41271e0d", - "is_block_valid": true, - "invalid_reason": null, - "timestamp": "1509977988" - }, - "signature": "3tW2EBVgxaZTE6nixVd9QEQf1vUxqPmQaNAMdCHc7zHik5KEosdkwScGYt4VhiHDTB6BCxTUzmqu3P7oP93tRWfj" - } diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 533fd802..b124a598 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -6,12 +6,6 @@ Libraries and Tools Maintained by the BigchainDB Team * `Python Driver `_ * `JavaScript / Node.js Driver `_ -* `The Transaction CLI `_ is - a command-line interface for building BigchainDB transactions. - You may be able to call it from inside the language of - your choice, and then use :ref:`the HTTP API ` - to post transactions. - Community-Driven Libraries and Tools ------------------------------------ diff --git a/docs/server/source/events/websocket-event-stream-api.rst b/docs/server/source/events/websocket-event-stream-api.rst index 850aded0..b0c83393 100644 --- a/docs/server/source/events/websocket-event-stream-api.rst +++ b/docs/server/source/events/websocket-event-stream-api.rst @@ -71,7 +71,7 @@ All messages sent in a stream are in the JSON format. For simplicity, BigchainDB initially only provides a stream for all validated transactions. In the future, we may provide streams for other - information, such as new blocks, new votes, or invalid transactions. We may + information. We may also provide the ability to filter the stream for specific qualities, such as a specific ``output``'s ``public_key``. @@ -101,8 +101,4 @@ Example message: .. note:: Transactions in BigchainDB are validated in batches ("blocks") and will, - therefore, be streamed in batches. Each block can contain up to a 1000 - transactions, ordered by the time at which they were included in the block. - The ``/valid_transactions`` stream will send these transactions in the same - order that the block stored them in, but this does **NOT** guarantee that - you will recieve the events in that same order. + therefore, be streamed in batches. diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index d9116e25..461efa83 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -31,7 +31,7 @@ with something like the following in the body: .. _api-root-endpoint: API Root Endpoint -------------------- +----------------- If you send an HTTP GET request to the API Root Endpoint e.g. ``http://localhost:9984/api/v1/`` @@ -44,7 +44,7 @@ that allows you to discover the BigchainDB API endpoints: Transactions -------------------- +------------ .. http:get:: /api/v1/transactions/{transaction_id} @@ -138,10 +138,6 @@ Transactions .. note:: - This option is only available when using BigchainDB with Tendermint. - - .. note:: - The posted `transaction `_ should be structurally valid and not spending an already spent output. @@ -163,15 +159,12 @@ Transactions :language: http .. note:: - If the server is returning a ``202`` HTTP status code, then the - transaction has been accepted for processing. To check the status of the - transaction, poll the link to the - :ref:`status monitor ` - provided in the ``Location`` header or listen to server's - :ref:`WebSocket Event Stream API `. + + If the server is returning a ``202`` HTTP status code when ``mode=aysnc`` or ``mode=sync``, then the + transaction has been accepted for processing. The client can subscribe to the + :ref:`WebSocket Event Stream API ` to listen for comitted transactions. :resheader Content-Type: ``application/json`` - :resheader Location: Relative link to a status monitor for the submitted transaction. :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. @@ -179,7 +172,8 @@ Transactions .. http:post:: /api/v1/transactions - This endpoint (without any parameters) will push a new transaction. If BigchainDB is used with Tendermint, the default mode ``async`` is used. + This endpoint (without any parameters) will push a new transaction. + Since no ``mode`` parameter is included, the default mode is assumed: ``async``. Transaction Outputs @@ -295,71 +289,6 @@ unspent outputs. :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. -Statuses --------------------------------- - -.. http:get:: /api/v1/statuses - - Get the status of an asynchronously written transaction or block by their id. - - :query string transaction_id: transaction ID - :query string block_id: block ID - - .. note:: - - Exactly one of the ``transaction_id`` or ``block_id`` query parameters must be - used together with this endpoint (see below for getting `transaction - statuses <#get--statuses?tx_id=tx_id>`_ and `block statuses - <#get--statuses?block_id=block_id>`_). - -.. _get_status_of_transaction: - -.. http:get:: /api/v1/statuses?transaction_id={transaction_id} - - Get the status of a transaction. - - The possible status values are ``undecided``, ``valid`` or ``backlog``. - If a transaction in neither of those states is found, a ``404 Not Found`` - HTTP status code is returned. `We're currently looking into ways to unambigously let the user know about a transaction's status that was included in an invalid block. `_ - - **Example request**: - - .. literalinclude:: http-samples/get-statuses-tx-request.http - :language: http - - **Example response**: - - .. literalinclude:: http-samples/get-statuses-tx-valid-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A transaction with that ID was found. - :statuscode 404: A transaction with that ID was not found. - - -.. http:get:: /api/v1/statuses?block_id={block_id} - - Get the status of a block. - - The possible status values are ``undecided``, ``valid`` or ``invalid``. - - **Example request**: - - .. literalinclude:: http-samples/get-statuses-block-request.http - :language: http - - **Example response**: - - .. literalinclude:: http-samples/get-statuses-block-valid-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A block with that ID was found. - :statuscode 404: A block with that ID was not found. - - Assets -------------------------------- @@ -594,9 +523,6 @@ a certain transaction with ``transaction_id`` occured in (a transaction can occu either gets rejected or validated by the system). This endpoint gives the ability to drill down on the lifecycle of a transaction -The `votes endpoint <#votes>`_ contains all the voting information for a specific block. So after retrieving the -``block_id`` for a given ``transaction_id``, one can now simply inspect the votes that happened at a specific time on that block. - Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -682,37 +608,6 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``transaction_id``. -Votes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. http:get:: /api/v1/votes?block_id={block_id} - - Retrieve a list of votes for a certain block with ID ``block_id``. - To check for the validity of a vote, a user of this endpoint needs to - perform the `following steps: `_ - - 1. Check if the vote's ``node_pubkey`` is allowed to vote. - 2. Verify the vote's signature against the vote's body (``vote.vote``) and ``node_pubkey``. - - - :query string block_id: The block ID to filter the votes. - - **Example request**: - - .. literalinclude:: http-samples/get-vote-request.http - :language: http - - **Example response**: - - .. literalinclude:: http-samples/get-vote-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A list of votes voting for a block with ID ``block_id`` was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/votes``, without defining ``block_id``. - - .. _determining-the-api-root-url: Determining the API Root URL diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index b7c5526e..e6f4c296 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -48,7 +48,6 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_DATABASE_KEYFILE`
`BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE`
`BIGCHAINDB_DATABASE_CRLFILE`
-`BIGCHAINDB_GRAPHITE_HOST`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -217,7 +216,7 @@ export BIGCHAINDB_SERVER_WORKERS=5 ## wsserver.scheme, wsserver.host and wsserver.port These settings are for the -[aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), +[aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), which is used to serve the [WebSocket Event Stream API](../websocket-event-stream-api.html). `wsserver.scheme` should be either `"ws"` or `"wss"` @@ -360,8 +359,8 @@ The full paths to the files where logs and error logs should be written to. } ``` -**Defaults to**: - +**Defaults to**: + * `"~/bigchaindb.log"` * `"~/bigchaindb-errors.log"` @@ -383,7 +382,7 @@ For example if we consider the log file setting: ``` logs would always be written to `bigchain.log`. Each time the file -`bigchain.log` reaches 200 MB it would be closed and renamed +`bigchain.log` reaches 200 MB it would be closed and renamed `bigchain.log.1`. If `bigchain.log.1` and `bigchain.log.2` already exist they would be renamed `bigchain.log.2` and `bigchain.log.3`. This pattern would be applied up to `bigchain.log.5` after which `bigchain.log.5` would be @@ -550,29 +549,3 @@ The port number at which the logging server should listen. ``` **Defaults to**: `9020` - - -## graphite.host - -The host name or IP address of a server listening for statsd events on UDP -port 8125. This defaults to `localhost`, and if no statsd collector is running, -the events are simply dropped by the operating system. - -**Example using environment variables** -```text -export BIGCHAINDB_GRAPHITE_HOST=10.0.0.5 -``` - -**Example config file snippet** -```js -"graphite": { - "host": "10.0.0.5" -} -``` - -**Default values (from a config file)** -```js -"graphite": { - "host": "localhost" -} -``` diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md deleted file mode 100644 index befe2400..00000000 --- a/scripts/benchmarks/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Benchmarks - -## CREATE transaction throughput - -This is a measurement of the throughput of CREATE transactions through the entire -pipeline, ie, the web frontend, block creation, and block validation, where the -output of the measurement is transactions per second. - -The benchmark runs for a fixed period of time and makes metrics available via -a graphite interface. - -### Running the benchmark - -Dependencies: - -* Python 3.5+ -* docker-compose 1.8.0+ -* docker 1.12+ - -To start: - - $ python3 scripts/benchmarks/create_thoughtput.py - -To start using a separate namespace for docker-compose: - - $ COMPOSE_PROJECT_NAME=somename python3 scripts/benchmarks/create_thoughtput.py - -### Results - -A test was run on AWS with the following instance configuration: - -* Ubuntu Server 16.04 (ami-060cde69) -* 32 core compute optimized (c3.8xlarge) -* 100gb root volume (300/3000 IOPS) - -The server received and validated over 800 transactions per second: - -![BigchainDB transaction throughput](https://cloud.githubusercontent.com/assets/125019/26688641/85d56d1e-46f3-11e7-8148-bf3bc8c54c33.png) - -For more information on how the benchmark was run, the abridged session buffer [is available](https://gist.github.com/libscott/8a37c5e134b2d55cfb55082b1cd85a02). diff --git a/scripts/benchmarks/create_thoughtput.py b/scripts/benchmarks/create_thoughtput.py deleted file mode 100644 index 2fe8c557..00000000 --- a/scripts/benchmarks/create_thoughtput.py +++ /dev/null @@ -1,133 +0,0 @@ -import sys -import math -import time -import requests -import subprocess -import multiprocessing - - -def main(): - cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d mdb') - cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d bdb') - cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d graphite') - - out = cmd('docker-compose -f benchmark.yml port graphite 80', capture=True) - graphite_web = 'http://localhost:%s/' % out.strip().split(':')[1] - print('Graphite web interface at: ' + graphite_web) - - start = time.time() - - cmd('docker-compose -f docker-compose.yml -f benchmark.yml exec bdb python %s load' % sys.argv[0]) - - mins = math.ceil((time.time() - start) / 60) + 1 - - graph_url = graphite_web + 'render/?width=900&height=600&_salt=1495462891.335&target=stats.pipelines.block.throughput&target=stats.pipelines.vote.throughput&target=stats.web.tx.post&from=-%sminutes' % mins # noqa - - print(graph_url) - - -def load(): - from bigchaindb.core import Bigchain - from bigchaindb.common.crypto import generate_key_pair - from bigchaindb.common.transaction import Transaction - - def transactions(): - priv, pub = generate_key_pair() - tx = Transaction.create([pub], [([pub], 1)]) - while True: - i = yield tx.to_dict() - tx.asset = {'data': {'n': i}} - tx.sign([priv]) - - def wait_for_up(): - print('Waiting for server to start... ', end='') - while True: - try: - requests.get('http://localhost:9984/') - break - except requests.ConnectionError: - time.sleep(0.1) - print('Ok') - - def post_txs(): - txs = transactions() - txs.send(None) - try: - with requests.Session() as session: - while True: - i = tx_queue.get() - if i is None: - break - tx = txs.send(i) - res = session.post('http://localhost:9984/api/v1/transactions/', json=tx) - assert res.status_code == 202 - except KeyboardInterrupt: - pass - - wait_for_up() - num_clients = 30 - test_time = 60 - tx_queue = multiprocessing.Queue(maxsize=num_clients) - txn = 0 - b = Bigchain() - - start_time = time.time() - - for i in range(num_clients): - multiprocessing.Process(target=post_txs).start() - - print('Sending transactions') - while time.time() - start_time < test_time: - # Post 500 transactions to the server - for i in range(500): - tx_queue.put(txn) - txn += 1 - print(txn) - while True: - # Wait for the server to reduce the backlog to below - # 10000 transactions. The expectation is that 10000 transactions - # will not be processed faster than a further 500 transactions can - # be posted, but nonetheless will be processed within a few seconds. - # This keeps the test from running on and keeps the transactions from - # being considered stale. - count = b.connection.db.backlog.count() - if count > 10000: - time.sleep(0.2) - else: - break - - for i in range(num_clients): - tx_queue.put(None) - - print('Waiting to clear backlog') - while True: - bl = b.connection.db.backlog.count() - if bl == 0: - break - print(bl) - time.sleep(1) - - print('Waiting for all votes to come in') - while True: - blocks = b.connection.db.bigchain.count() - votes = b.connection.db.votes.count() - if blocks == votes + 1: - break - print('%s blocks, %s votes' % (blocks, votes)) - time.sleep(3) - - print('Finished') - - -def cmd(command, capture=False): - stdout = subprocess.PIPE if capture else None - args = ['bash', '-c', command] - proc = subprocess.Popen(args, stdout=stdout) - assert not proc.wait() - return capture and proc.stdout.read().decode() - - -if sys.argv[1:] == ['load']: - load() -else: - main() diff --git a/setup.py b/setup.py index c3fdde9d..f15df505 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ tests_require = [ 'pytest-xdist', 'pytest-flask', 'pytest-aiohttp', + 'pytest-asyncio', 'tox', ] + docs_require @@ -83,7 +84,6 @@ install_requires = [ 'pyyaml~=3.12', 'aiohttp~=2.3', 'python-rapidjson-schema==0.1.1', - 'statsd==3.2.1', 'abci~=0.3.0', ] diff --git a/tests/backend/localmongodb/conftest.py b/tests/backend/localmongodb/conftest.py new file mode 100644 index 00000000..b51a0ebb --- /dev/null +++ b/tests/backend/localmongodb/conftest.py @@ -0,0 +1,12 @@ +from pymongo import MongoClient +from pytest import fixture + + +@fixture +def mongo_client(db_context): + return MongoClient(host=db_context.host, port=db_context.port) + + +@fixture +def utxo_collection(db_context, mongo_client): + return mongo_client[db_context.name].utxos diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index 75f5190c..ec483793 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -6,6 +6,23 @@ import pymongo pytestmark = [pytest.mark.tendermint, pytest.mark.localmongodb, pytest.mark.bdb] +@pytest.fixture +def dummy_unspent_outputs(): + return [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 1}, + {'transaction_id': 'b', 'output_index': 0}, + ] + + +@pytest.fixture +def utxoset(dummy_unspent_outputs, utxo_collection): + insert_res = utxo_collection.insert_many(deepcopy(dummy_unspent_outputs)) + assert insert_res.acknowledged + assert len(insert_res.inserted_ids) == 3 + return dummy_unspent_outputs, utxo_collection + + def test_get_txids_filtered(signed_create_tx, signed_transfer_tx): from bigchaindb.backend import connect, query from bigchaindb.models import Transaction @@ -180,3 +197,96 @@ def test_get_block(): block = dict(query.get_block(conn, 3)) assert block['height'] == 3 + + +def test_delete_zombie_transactions(signed_create_tx, signed_transfer_tx): + from bigchaindb.backend import connect, query + from bigchaindb.tendermint.lib import Block + conn = connect() + + conn.db.transactions.insert_one(signed_create_tx.to_dict()) + query.store_asset(conn, {'id': signed_create_tx.id}) + block = Block(app_hash='random_utxo', + height=3, + transactions=[signed_create_tx.id]) + query.store_block(conn, block._asdict()) + + conn.db.transactions.insert_one(signed_transfer_tx.to_dict()) + query.store_metadatas(conn, [{'id': signed_transfer_tx.id}]) + + query.delete_zombie_transactions(conn) + assert query.get_transaction(conn, signed_transfer_tx.id) is None + assert query.get_asset(conn, signed_transfer_tx.id) is None + assert list(query.get_metadata(conn, [signed_transfer_tx.id])) == [] + + assert query.get_transaction(conn, signed_create_tx.id) is not None + assert query.get_asset(conn, signed_create_tx.id) is not None + + +def test_delete_latest_block(signed_create_tx, signed_transfer_tx): + from bigchaindb.backend import connect, query + from bigchaindb.tendermint.lib import Block + conn = connect() + + conn.db.transactions.insert_one(signed_create_tx.to_dict()) + query.store_asset(conn, {'id': signed_create_tx.id}) + block = Block(app_hash='random_utxo', + height=51, + transactions=[signed_create_tx.id]) + query.store_block(conn, block._asdict()) + query.delete_latest_block(conn) + + assert query.get_transaction(conn, signed_create_tx.id) is None + assert query.get_asset(conn, signed_create_tx.id) is None + assert query.get_block(conn, 51) is None + + +def test_delete_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + unspent_outputs, utxo_collection = utxoset + delete_res = query.delete_unspent_outputs(db_context.conn, + *unspent_outputs[::2]) + assert delete_res['n'] == 2 + assert utxo_collection.find( + {'$or': [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'b', 'output_index': 0}, + ]} + ).count() == 0 + assert utxo_collection.find( + {'transaction_id': 'a', 'output_index': 1}).count() == 1 + + +def test_store_one_unspent_output(db_context, + unspent_output_1, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn, unspent_output_1) + assert res.acknowledged + assert len(res.inserted_ids) == 1 + assert utxo_collection.find( + {'transaction_id': unspent_output_1['transaction_id'], + 'output_index': unspent_output_1['output_index']} + ).count() == 1 + + +def test_store_many_unspent_outputs(db_context, + unspent_outputs, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn, *unspent_outputs) + assert res.acknowledged + assert len(res.inserted_ids) == 3 + assert utxo_collection.find( + {'transaction_id': unspent_outputs[0]['transaction_id']} + ).count() == 3 + + +def test_get_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + cursor = query.get_unspent_outputs(db_context.conn) + assert cursor.count() == 3 + retrieved_utxoset = list(cursor) + unspent_outputs, utxo_collection = utxoset + assert retrieved_utxoset == list(utxo_collection.find()) + for utxo in retrieved_utxoset: + del utxo['_id'] + assert retrieved_utxoset == unspent_outputs diff --git a/tests/backend/localmongodb/test_schema.py b/tests/backend/localmongodb/test_schema.py new file mode 100644 index 00000000..beee421c --- /dev/null +++ b/tests/backend/localmongodb/test_schema.py @@ -0,0 +1,110 @@ +import pytest + + +pytestmark = [pytest.mark.bdb, pytest.mark.tendermint] + + +def test_init_creates_db_tables_and_indexes(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend.schema import init_database + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # the db is set up by the fixture so we need to remove it + conn.conn.drop_database(dbname) + + init_database() + + collection_names = conn.conn[dbname].collection_names() + assert set(collection_names) == { + 'transactions', 'assets', 'metadata', 'blocks', 'utxos'} + + indexes = conn.conn[dbname]['assets'].index_information().keys() + assert set(indexes) == {'_id_', 'asset_id', 'text'} + + indexes = conn.conn[dbname]['transactions'].index_information().keys() + assert set(indexes) == { + '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} + + indexes = conn.conn[dbname]['blocks'].index_information().keys() + assert set(indexes) == {'_id_', 'height'} + + indexes = conn.conn[dbname]['utxos'].index_information().keys() + assert set(indexes) == {'_id_', 'utxo'} + + +def test_init_database_fails_if_db_exists(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend.schema import init_database + from bigchaindb.common import exceptions + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures + assert dbname in conn.conn.database_names() + + with pytest.raises(exceptions.DatabaseAlreadyExists): + init_database() + + +def test_create_tables(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures so we need to remove it + conn.conn.drop_database(dbname) + schema.create_database(conn, dbname) + schema.create_tables(conn, dbname) + + collection_names = conn.conn[dbname].collection_names() + assert set(collection_names) == { + 'transactions', 'assets', 'metadata', 'blocks', 'utxos'} + + +def test_create_secondary_indexes(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures so we need to remove it + conn.conn.drop_database(dbname) + schema.create_database(conn, dbname) + schema.create_tables(conn, dbname) + schema.create_indexes(conn, dbname) + + indexes = conn.conn[dbname]['assets'].index_information().keys() + assert set(indexes) == {'_id_', 'asset_id', 'text'} + + indexes = conn.conn[dbname]['transactions'].index_information().keys() + assert set(indexes) == { + '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} + + indexes = conn.conn[dbname]['blocks'].index_information().keys() + assert set(indexes) == {'_id_', 'height'} + + index_info = conn.conn[dbname]['utxos'].index_information() + assert set(index_info.keys()) == {'_id_', 'utxo'} + assert index_info['utxo']['unique'] + assert index_info['utxo']['key'] == [('transaction_id', 1), + ('output_index', 1)] + + +def test_drop(dummy_db): + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + assert dummy_db in conn.conn.database_names() + schema.drop_database(conn, dummy_db) + assert dummy_db not in conn.conn.database_names() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 423e4614..aa0df51e 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -6,14 +6,15 @@ import copy import pytest +@pytest.mark.tendermint def test_make_sure_we_dont_remove_any_command(): # thanks to: http://stackoverflow.com/a/18161115/597097 from bigchaindb.commands.bigchaindb import create_parser parser = create_parser() - assert parser.parse_args(['configure', 'rethinkdb']).command - assert parser.parse_args(['configure', 'mongodb']).command + assert parser.parse_args(['configure', 'localmongodb']).command + assert parser.parse_args(['configure', 'localmongodb']).command assert parser.parse_args(['show-config']).command assert parser.parse_args(['export-my-pubkey']).command assert parser.parse_args(['init']).command @@ -517,3 +518,99 @@ def test_run_remove_replicas(mock_remove_replicas): assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 mock_remove_replicas.reset_mock() + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_recover_db_from_zombie_txn(b, monkeypatch): + from bigchaindb.commands.bigchaindb import run_recover + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.tendermint.lib import Block + from bigchaindb import backend + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset={'cycle': 'hero'}, + metadata={'name': 'hohenheim'}) \ + .sign([alice.private_key]) + b.store_bulk_transactions([tx]) + block = Block(app_hash='random_app_hash', height=10, + transactions=[])._asdict() + b.store_block(block) + + def mock_get(uri): + return MockResponse(10) + monkeypatch.setattr('requests.get', mock_get) + + run_recover(b) + + assert list(backend.query.get_metadata(b.connection, [tx.id])) == [] + assert not backend.query.get_asset(b.connection, tx.id) + assert not b.get_transaction(tx.id) + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_recover_db_from_zombie_block(b, monkeypatch): + from bigchaindb.commands.bigchaindb import run_recover + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.tendermint.lib import Block + from bigchaindb import backend + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset={'cycle': 'hero'}, + metadata={'name': 'hohenheim'}) \ + .sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + block9 = Block(app_hash='random_app_hash', height=9, + transactions=[])._asdict() + b.store_block(block9) + block10 = Block(app_hash='random_app_hash', height=10, + transactions=[tx.id])._asdict() + b.store_block(block10) + + def mock_get(uri): + return MockResponse(9) + monkeypatch.setattr('requests.get', mock_get) + + run_recover(b) + + assert list(backend.query.get_metadata(b.connection, [tx.id])) == [] + assert not backend.query.get_asset(b.connection, tx.id) + assert not b.get_transaction(tx.id) + + block = b.get_latest_block() + assert block['height'] == 9 + + +@pytest.mark.tendermint +@patch('bigchaindb.config_utils.autoconfigure') +@patch('bigchaindb.commands.bigchaindb.run_recover') +@patch('bigchaindb.tendermint.commands.start') +def test_recover_db_on_start(mock_autoconfigure, + mock_run_recover, + mock_start, + mocked_setup_logging): + from bigchaindb.commands.bigchaindb import run_start + args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True, + skip_initialize_database=False) + run_start(args) + + assert mock_run_recover.called + assert mock_start.called + + +# Helper +class MockResponse(): + + def __init__(self, height): + self.height = height + + def json(self): + return {'result': {'latest_block_height': self.height}} diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 97cfcfb3..ae371adb 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -343,7 +343,7 @@ def tri_state_transaction(request): 'uri': 'ni:///sha-256;49C5UWNODwtcINxLgLc90bMCFqCymFYONGEmV4a0sG4?fpt=ed25519-sha-256&cost=131072'}, 'public_keys': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] }], - 'version': '1.0' + 'version': '2.0' } tx['id'] = request.param['id'] tx['inputs'][0]['fulfillment'] = request.param['fulfillment'] diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 39048e19..2617197d 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -5,9 +5,12 @@ import json from copy import deepcopy from base58 import b58encode, b58decode +from cryptoconditions import Ed25519Sha256 from pytest import mark, raises from sha3 import sha3_256 +pytestmark = mark.tendermint + def test_input_serialization(ffill_uri, user_pub): from bigchaindb.common.transaction import Input @@ -1005,3 +1008,42 @@ def test_output_from_dict_invalid_amount(user_output): out['amount'] = 'a' with raises(AmountError): Output.from_dict(out) + + +def test_unspent_outputs_property(merlin, alice, bob, carol): + from bigchaindb.common.transaction import Transaction + tx = Transaction.create( + [merlin.public_key], + [([alice.public_key], 1), + ([bob.public_key], 2), + ([carol.public_key], 3)], + asset={'hash': '06e47bcf9084f7ecfd2a2a2ad275444a'}, + ).sign([merlin.private_key]) + unspent_outputs = list(tx.unspent_outputs) + assert len(unspent_outputs) == 3 + assert all(utxo.transaction_id == tx.id for utxo in unspent_outputs) + assert all(utxo.asset_id == tx.id for utxo in unspent_outputs) + assert all( + utxo.output_index == i for i, utxo in enumerate(unspent_outputs)) + unspent_output_0 = unspent_outputs[0] + assert unspent_output_0.amount == 1 + assert unspent_output_0.condition_uri == Ed25519Sha256( + public_key=b58decode(alice.public_key)).condition_uri + unspent_output_1 = unspent_outputs[1] + assert unspent_output_1.amount == 2 + assert unspent_output_1.condition_uri == Ed25519Sha256( + public_key=b58decode(bob.public_key)).condition_uri + unspent_output_2 = unspent_outputs[2] + assert unspent_output_2.amount == 3 + assert unspent_output_2.condition_uri == Ed25519Sha256( + public_key=b58decode(carol.public_key)).condition_uri + + +def test_spent_outputs_property(signed_transfer_tx): + spent_outputs = list(signed_transfer_tx.spent_outputs) + tx = signed_transfer_tx.to_dict() + assert len(spent_outputs) == 1 + spent_output = spent_outputs[0] + assert spent_output['transaction_id'] == tx['inputs'][0]['fulfills']['transaction_id'] + assert spent_output['output_index'] == tx['inputs'][0]['fulfills']['output_index'] + # assert spent_output._asdict() == tx['inputs'][0]['fulfills'] diff --git a/tests/conftest.py b/tests/conftest.py index 90894491..3899e71d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -155,6 +155,7 @@ def _configure_bigchaindb(request, ssl_context): } } config['database']['name'] = test_db_name + config = config_utils.env_config(config) config_utils.set_config(config) @@ -309,10 +310,26 @@ def carol_pubkey(carol): return carol.public_key +@pytest.fixture +def merlin(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def merlin_privkey(merlin): + return merlin.private_key + + +@pytest.fixture +def merlin_pubkey(merlin): + return merlin.public_key + + @pytest.fixture def b(): - from bigchaindb import Bigchain - return Bigchain() + from bigchaindb.tendermint import BigchainDB + return BigchainDB() @pytest.fixture @@ -499,6 +516,32 @@ def db_context(db_config, db_host, db_port, db_name, db_conn): ) +@pytest.fixture +def tendermint_host(): + return os.getenv('BIGCHAINDB_TENDERMINT_HOST', 'localhost') + + +@pytest.fixture +def tendermint_port(): + return int(os.getenv('BIGCHAINDB_TENDERMINT_PORT', 46657)) + + +@pytest.fixture +def tendermint_ws_url(tendermint_host, tendermint_port): + return f'ws://{tendermint_host}:{tendermint_port}/websocket' + + +@pytest.fixture +def tendermint_context(tendermint_host, tendermint_port, tendermint_ws_url): + TendermintContext = namedtuple( + 'TendermintContext', ('host', 'port', 'ws_url')) + return TendermintContext( + host=tendermint_host, + port=tendermint_port, + ws_url=tendermint_ws_url, + ) + + @pytest.fixture def mocked_setup_pub_logger(mocker): return mocker.patch( @@ -591,3 +634,44 @@ def genesis_tx(b, user_pk): tx.operation = Transaction.GENESIS genesis_tx = tx.sign([b.me_private]) return genesis_tx + + +@pytest.fixture +def unspent_output_0(): + return { + 'amount': 1, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 0, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d' + } + + +@pytest.fixture +def unspent_output_1(): + return { + 'amount': 2, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 1, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + } + + +@pytest.fixture +def unspent_output_2(): + return { + 'amount': 3, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 2, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + } + + +@pytest.fixture +def unspent_outputs(unspent_output_0, unspent_output_1, unspent_output_2): + return unspent_output_0, unspent_output_1, unspent_output_2 diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py index 56c871e6..7456f130 100644 --- a/tests/tendermint/test_core.py +++ b/tests/tendermint/test_core.py @@ -57,8 +57,8 @@ def test_deliver_tx__valid_create_updates_db(b): .sign([alice.private_key]) app = App(b) - app.init_chain(["ignore"]) - app.begin_block("ignore") + app.init_chain(['ignore']) + app.begin_block('ignore') result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_ok() @@ -81,8 +81,8 @@ def test_deliver_tx__double_spend_fails(b): .sign([alice.private_key]) app = App(b) - app.init_chain(["ignore"]) - app.begin_block("ignore") + app.init_chain(['ignore']) + app.begin_block('ignore') result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_ok() @@ -101,8 +101,8 @@ def test_deliver_transfer_tx__double_spend_fails(b): from bigchaindb.common.crypto import generate_key_pair app = App(b) - app.init_chain(["ignore"]) - app.begin_block("ignore") + app.init_chain(['ignore']) + app.begin_block('ignore') alice = generate_key_pair() bob = generate_key_pair() diff --git a/tests/tendermint/test_event_stream.py b/tests/tendermint/test_event_stream.py index da5c4c6f..345f653c 100644 --- a/tests/tendermint/test_event_stream.py +++ b/tests/tendermint/test_event_stream.py @@ -1,9 +1,13 @@ +import json from queue import Queue +from aiohttp import ClientSession import pytest -@pytest.mark.tendermint +pytestmark = pytest.mark.tendermint + + def test_process_event_new_block(): from bigchaindb.tendermint.event_stream import process_event @@ -33,7 +37,6 @@ def test_process_event_new_block(): assert not event_queue.empty() -@pytest.mark.tendermint def test_process_event_empty_block(): from bigchaindb.tendermint.event_stream import process_event @@ -52,7 +55,6 @@ def test_process_event_empty_block(): assert event_queue.empty() -@pytest.mark.tendermint def test_process_unknown_event(): from bigchaindb.tendermint.event_stream import process_event @@ -62,3 +64,18 @@ def test_process_unknown_event(): event_queue = Queue() process_event(event_queue, event, 'test_stream_id') assert event_queue.empty() + + +@pytest.mark.asyncio +async def test_subscribe_events(tendermint_ws_url): + from bigchaindb.tendermint.event_stream import subscribe_events + session = ClientSession() + ws = await session.ws_connect(tendermint_ws_url) + stream_id = 'bigchaindb_stream_01' + await subscribe_events(ws, stream_id) + msg = await ws.receive() + assert msg.data + msg_data_dict = json.loads(msg.data) + assert msg_data_dict['id'] == stream_id + assert msg_data_dict['jsonrpc'] == '2.0' + # TODO What else should be there? Right now, getting error. diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index dd6aef4a..f156c871 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -304,7 +304,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request, ssl_con 'granular_levels': {}, 'port': 9020 }, - 'graphite': {'host': 'localhost'}, } diff --git a/tests/test_core.py b/tests/test_core.py index 7a369bdd..b18c56d3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -24,7 +24,6 @@ def config(request, monkeypatch): 'keyring': [], 'CONFIGURED': True, 'backlog_reassign_delay': 30, - 'graphite': {'host': 'localhost'}, } monkeypatch.setattr('bigchaindb.config', config) diff --git a/tests/utils.py b/tests/utils.py index 50dedae9..d8784809 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,6 +46,7 @@ def flush_mongo_db(connection, dbname): connection.conn[dbname].votes.delete_many({}) connection.conn[dbname].assets.delete_many({}) connection.conn[dbname].metadata.delete_many({}) + connection.conn[dbname].utxos.delete_many({}) @flush_db.register(LocalMongoDBConnection) @@ -55,6 +56,7 @@ def flush_localmongo_db(connection, dbname): connection.conn[dbname].transactions.delete_many({}) connection.conn[dbname].assets.delete_many({}) connection.conn[dbname].metadata.delete_many({}) + connection.conn[dbname].utxos.delete_many({}) @singledispatch diff --git a/tests/validation/test_transaction_structure.py b/tests/validation/test_transaction_structure.py index 5b128003..35f86cd1 100644 --- a/tests/validation/test_transaction_structure.py +++ b/tests/validation/test_transaction_structure.py @@ -237,7 +237,7 @@ def test_unsupported_condition_type(): # Version def test_validate_version(b, create_tx): - create_tx.version = '1.0' + create_tx.version = '2.0' create_tx.sign([b.me_private]) validate(create_tx) diff --git a/tests/web/test_info.py b/tests/web/test_info.py index 459f3e05..f682b5e3 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -13,7 +13,6 @@ def test_api_root_endpoint(client, wsserver_base_url): 'v1': { 'docs': ''.join(docs_url), 'transactions': '/api/v1/transactions/', - 'statuses': '/api/v1/statuses/', 'assets': '/api/v1/assets/', 'outputs': '/api/v1/outputs/', 'streams': '{}/api/v1/streams/valid_transactions'.format( @@ -37,7 +36,6 @@ def test_api_v1_endpoint(client, wsserver_base_url): api_v1_info = { 'docs': ''.join(docs_url), 'transactions': '/transactions/', - 'statuses': '/statuses/', 'assets': '/assets/', 'outputs': '/outputs/', 'streams': '{}/api/v1/streams/valid_transactions'.format( diff --git a/tests/web/test_statuses.py b/tests/web/test_statuses.py deleted file mode 100644 index 7c2a7c14..00000000 --- a/tests/web/test_statuses.py +++ /dev/null @@ -1,94 +0,0 @@ -import pytest - -from bigchaindb.models import Transaction - -STATUSES_ENDPOINT = '/api/v1/statuses' - - -@pytest.mark.bdb -@pytest.mark.usefixtures('inputs') -def test_get_transaction_status_endpoint(b, client, user_pk): - input_tx = b.get_owned_ids(user_pk).pop() - tx, status = b.get_transaction(input_tx.txid, include_status=True) - res = client.get(STATUSES_ENDPOINT + '?transaction_id=' + input_tx.txid) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -def test_get_transaction_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + '?transaction_id=123') - assert res.status_code == 404 - - -@pytest.mark.bdb -def test_get_block_status_endpoint_undecided(b, client): - tx = Transaction.create([b.me], [([b.me], 1)]) - tx = tx.sign([b.me_private]) - - block = b.create_block([tx]) - b.write_block(block) - - status = b.block_election_status(block) - - res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -@pytest.mark.usefixtures('inputs') -def test_get_block_status_endpoint_valid(b, client): - tx = Transaction.create([b.me], [([b.me], 1)]) - tx = tx.sign([b.me_private]) - - block = b.create_block([tx]) - b.write_block(block) - - # vote the block valid - vote = b.vote(block.id, b.get_last_voted_block().id, True) - b.write_vote(vote) - - status = b.block_election_status(block) - - res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -@pytest.mark.usefixtures('inputs') -def test_get_block_status_endpoint_invalid(b, client): - tx = Transaction.create([b.me], [([b.me], 1)]) - tx = tx.sign([b.me_private]) - - block = b.create_block([tx]) - b.write_block(block) - - # vote the block valid - vote = b.vote(block.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - - status = b.block_election_status(block) - - res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) - assert status == res.json['status'] - assert res.status_code == 200 - - -@pytest.mark.bdb -def test_get_block_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + '?block_id=123') - assert res.status_code == 404 - - -@pytest.mark.bdb -def test_get_status_endpoint_returns_400_bad_query_params(client): - res = client.get(STATUSES_ENDPOINT) - assert res.status_code == 400 - - res = client.get(STATUSES_ENDPOINT + '?ts_id=123') - assert res.status_code == 400 - - res = client.get(STATUSES_ENDPOINT + '?transaction_id=123&block_id=123') - assert res.status_code == 400 diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index a67d790e..f376bf09 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -44,9 +44,6 @@ def test_post_create_transaction_endpoint(b, client): assert res.status_code == 202 - assert '../statuses?transaction_id={}'.format(tx.id) in \ - res.headers['Location'] - assert res.json['inputs'][0]['owners_before'][0] == user_pub assert res.json['outputs'][0]['public_keys'][0] == user_pub diff --git a/tmdata/genesis.json b/tmdata/genesis.json index 1c5ae703..877500df 100644 --- a/tmdata/genesis.json +++ b/tmdata/genesis.json @@ -1 +1 @@ -{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-EhS6zg","validators":[{"pub_key":{"type":"ed25519","data":"C3E96823EB67401C5B794F4100CEA04B745C29A0707979485EAE4F3C1A7D8583"},"power":10,"name":""}],"app_hash":""} \ No newline at end of file +{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-JCYeEN","validators":[{"pub_key":{"type":"ed25519","data":"0C988282C02CFF72E5E296DB78CE26D922178549B327C375D992548C9AFCCE6D"},"power":10,"name":""}],"app_hash":""} diff --git a/tmdata/priv_validator.json b/tmdata/priv_validator.json index d48911de..1dc74a58 100644 --- a/tmdata/priv_validator.json +++ b/tmdata/priv_validator.json @@ -1 +1 @@ -{"address":"2571CF34499E67BED107A6FE73418667A4461F6A","pub_key":{"type":"ed25519","data":"C3E96823EB67401C5B794F4100CEA04B745C29A0707979485EAE4F3C1A7D8583"},"last_height":1,"last_round":0,"last_step":3,"last_signature":{"type":"ed25519","data":"C9C16BEB7C71DC6FFDFE06973BEFDD995BD12F5A3B124F387E037C6E6F2CD913FA084495DEDEFABE5C52C385C8B909FDC0EB9BC15F3499B97C7693DCD1F82004"},"last_signbytes":"7B22636861696E5F6964223A22746573742D636861696E2D456853367A67222C22766F7465223A7B22626C6F636B5F6964223A7B2268617368223A2239334634373641364642323431393131343434324232444135434339303539353443393337373830222C227061727473223A7B2268617368223A2236423034414544374631323742353534433133383435394439413445423130394242354232413431222C22746F74616C223A317D7D2C22686569676874223A312C22726F756E64223A302C2274797065223A327D7D","priv_key":{"type":"ed25519","data":"5431111D9D8833305509308708341EBA0AD7226F30DBDA43616B4A73F94F2369C3E96823EB67401C5B794F4100CEA04B745C29A0707979485EAE4F3C1A7D8583"}} \ No newline at end of file +{"address":"E6CB05DA326F70BB4CC0A4AF83FC3BBF70B9A4D5","pub_key":{"type":"ed25519","data":"0C988282C02CFF72E5E296DB78CE26D922178549B327C375D992548C9AFCCE6D"},"last_height":0,"last_round":0,"last_step":0,"last_signature":null,"priv_key":{"type":"ed25519","data":"D4488996BDF92CE1D80670C66923D4996AE1B772FE0F76DAE33EDC410DC1D58F0C988282C02CFF72E5E296DB78CE26D922178549B327C375D992548C9AFCCE6D"}}