From 891289ed661ea064314171a87a6e25c84fdbd165 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 13 Apr 2016 16:47:18 +0200 Subject: [PATCH 01/11] Move monitoring code from the main class to processes --- bigchaindb/block.py | 16 +++++++++++----- bigchaindb/core.py | 13 ++----------- bigchaindb/monitor.py | 11 +++++++---- bigchaindb/voter.py | 8 +++++++- bigchaindb/web/server.py | 4 +++- bigchaindb/web/views.py | 23 +++++++++++++++++++---- 6 files changed, 49 insertions(+), 26 deletions(-) diff --git a/bigchaindb/block.py b/bigchaindb/block.py index 96da9207..2c2e108e 100644 --- a/bigchaindb/block.py +++ b/bigchaindb/block.py @@ -11,8 +11,6 @@ from bigchaindb.monitor import Monitor logger = logging.getLogger(__name__) -monitor = Monitor() - class Block(object): @@ -27,6 +25,7 @@ class Block(object): self.q_tx_delete = mp.Queue() self.q_block = mp.Queue() self.initialized = mp.Event() + self.monitor = Monitor() def filter_by_assignee(self): """ @@ -57,7 +56,9 @@ class Block(object): b = Bigchain() while True: - monitor.gauge('tx_queue_gauge', self.q_tx_to_validate.qsize(), rate=bigchaindb.config['statsd']['rate']) + self.monitor.gauge('tx_queue_gauge', + self.q_tx_to_validate.qsize(), + rate=bigchaindb.config['statsd']['rate']) tx = self.q_tx_to_validate.get() # poison pill @@ -67,7 +68,11 @@ class Block(object): return self.q_tx_delete.put(tx['id']) - if b.is_valid_transaction(tx): + + with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']): + is_valid_transaction = b.is_valid_transaction(tx) + + if is_valid_transaction: self.q_tx_validated.put(tx) def create_blocks(self): @@ -122,7 +127,8 @@ class Block(object): if block == 'stop': return - b.write_block(block) + with self.monitor.timer('write_block'): + b.write_block(block) def delete_transactions(self): """ diff --git a/bigchaindb/core.py b/bigchaindb/core.py index ea4b2576..0683170b 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,17 +1,13 @@ -import rethinkdb as r import random -import rapidjson +import rethinkdb as r +import rapidjson import bigchaindb from bigchaindb import util from bigchaindb import config_utils from bigchaindb import exceptions from bigchaindb import crypto -from bigchaindb.monitor import Monitor - - -monitor = Monitor() class GenesisBlockAlreadyExistsError(Exception): @@ -68,7 +64,6 @@ class Bigchain(object): def reconnect(self): return r.connect(host=self.host, port=self.port, db=self.dbname) - @monitor.timer('create_transaction', rate=bigchaindb.config['statsd']['rate']) def create_transaction(self, *args, **kwargs): """Create a new transaction @@ -104,7 +99,6 @@ class Bigchain(object): return self.consensus.verify_signature( signed_transaction, *args, **kwargs) - @monitor.timer('write_transaction', rate=bigchaindb.config['statsd']['rate']) def write_transaction(self, signed_transaction, durability='soft'): """Write the transaction to bigchain. @@ -239,7 +233,6 @@ class Bigchain(object): return owned - @monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']) def validate_transaction(self, transaction): """Validate a transaction. @@ -308,7 +301,6 @@ class Bigchain(object): return block - @monitor.timer('validate_block') # TODO: check that the votings structure is correctly constructed def validate_block(self, block): """Validate a block. @@ -351,7 +343,6 @@ class Bigchain(object): except Exception: return False - @monitor.timer('write_block') def write_block(self, block, durability='soft'): """Write a block to bigchain. diff --git a/bigchaindb/monitor.py b/bigchaindb/monitor.py index 9094fea3..3c14da1e 100644 --- a/bigchaindb/monitor.py +++ b/bigchaindb/monitor.py @@ -1,13 +1,14 @@ -import statsd from platform import node +import statsd + import bigchaindb from bigchaindb import config_utils -class Monitor(statsd.StatsClient): - """Set up statsd monitoring - """ +class Monitor(statsd.StatsClient): + """Set up statsd monitoring.""" + def __init__(self, *args, **kwargs): """Overrides statsd client, fixing prefix to messages and loading configuration @@ -15,6 +16,7 @@ class Monitor(statsd.StatsClient): *args: arguments (identical to Statsclient) **kwargs: keyword arguments (identical to Statsclient) """ + config_utils.autoconfigure() if not kwargs: @@ -28,3 +30,4 @@ class Monitor(statsd.StatsClient): if 'port' not in kwargs: kwargs['port'] = bigchaindb.config['statsd']['port'] super().__init__(*args, **kwargs) + diff --git a/bigchaindb/voter.py b/bigchaindb/voter.py index 3ed73636..9a34c0d8 100644 --- a/bigchaindb/voter.py +++ b/bigchaindb/voter.py @@ -3,6 +3,7 @@ import multiprocessing as mp import ctypes from bigchaindb import Bigchain +from bigchaindb.monitor import Monitor logger = logging.getLogger(__name__) @@ -56,6 +57,9 @@ class Voter(object): Initialize with a queue where new blocks added to the bigchain will be put """ + + self.monitor = Monitor() + self.q_new_block = q_new_block self.q_blocks_to_validate = mp.Queue() self.q_validated_block = mp.Queue() @@ -102,7 +106,9 @@ class Voter(object): logger.info('new_block arrived to voter') block_number = self.v_previous_block_number.value + 1 - validity = b.is_valid_block(new_block) + + with self.monitor.timer('validate_block'): + validity = b.is_valid_block(new_block) self.q_validated_block.put((new_block, self.v_previous_block_id.value.decode(), diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 021ceab4..b20ee049 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -7,10 +7,11 @@ import copy import multiprocessing from flask import Flask +import gunicorn.app.base from bigchaindb import Bigchain from bigchaindb.web import views -import gunicorn.app.base +from bigchaindb.monitor import Monitor class StandaloneApplication(gunicorn.app.base.BaseApplication): @@ -56,6 +57,7 @@ def create_app(debug=False): app = Flask(__name__) app.debug = debug app.config['bigchain'] = Bigchain() + app.config['monitor'] = Monitor() app.register_blueprint(views.basic_views, url_prefix='/api/v1') return app diff --git a/bigchaindb/web/views.py b/bigchaindb/web/views.py index 53db6f66..4a3b3691 100644 --- a/bigchaindb/web/views.py +++ b/bigchaindb/web/views.py @@ -7,20 +7,32 @@ For more information please refer to the documentation in Apiary: import flask from flask import current_app, request, Blueprint +import bigchaindb from bigchaindb import util basic_views = Blueprint('basic_views', __name__) +# Unfortunately I cannot find a reference to this decorator. +# This answer on SO is quite useful tho: +# - http://stackoverflow.com/a/13432373/597097 @basic_views.record -def get_bigchain(state): +def record(state): + """This function checks if the blueprint can be initialized + with the provided state.""" + bigchain = state.app.config.get('bigchain') + monitor = state.app.config.get('monitor') if bigchain is None: - raise Exception('This blueprint expects you to provide ' - 'database access through `bigchain`') + raise ValueError('This blueprint expects you to provide ' + 'database access through `bigchain`.') + if monitor is None: + raise ValueError('This blueprint expects you to provide ' + 'a monitor instance to record system ' + 'performance.') @basic_views.route('/transactions/') @@ -48,6 +60,7 @@ def create_transaction(): A JSON string containing the data about the transaction. """ bigchain = current_app.config['bigchain'] + monitor = current_app.config['monitor'] val = {} @@ -63,7 +76,9 @@ def create_transaction(): if not bigchain.consensus.verify_signature(tx): val['error'] = 'Invalid transaction signature' - val = bigchain.write_transaction(tx) + with monitor.timer('write_transaction', + rate=bigchaindb.config['statsd']['rate']): + val = bigchain.write_transaction(tx) return flask.jsonify(**tx) From d5af7a641d253a0fd69fddbbba59f1962ffa376b Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 14 Apr 2016 11:47:52 +0200 Subject: [PATCH 02/11] Use the client to do the loadtest --- bigchaindb/client.py | 3 +-- bigchaindb/commands/bigchain_benchmark.py | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/bigchaindb/client.py b/bigchaindb/client.py index 67ce51a9..fcf1a7f8 100644 --- a/bigchaindb/client.py +++ b/bigchaindb/client.py @@ -1,7 +1,6 @@ import requests import bigchaindb -from bigchaindb import util from bigchaindb import config_utils from bigchaindb import exceptions from bigchaindb import crypto @@ -112,5 +111,5 @@ def temp_client(): """ private_key, public_key = crypto.generate_key_pair() - return Client(private_key=private_key, public_key=public_key, api_endpoint='http://localhost:5000/api/v1') + return Client(private_key=private_key, public_key=public_key, api_endpoint=bigchaindb.config['api_endpoint']) diff --git a/bigchaindb/commands/bigchain_benchmark.py b/bigchaindb/commands/bigchain_benchmark.py index 2bac4c08..120987d6 100644 --- a/bigchaindb/commands/bigchain_benchmark.py +++ b/bigchaindb/commands/bigchain_benchmark.py @@ -7,23 +7,21 @@ import logstats import bigchaindb import bigchaindb.config_utils from bigchaindb.util import ProcessGroup +from bigchaindb.client import temp_client from bigchaindb.commands.utils import base_parser, start logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -USER_PUBLIC_KEY = 'qZAN9Ngs1v4qP1T5UBYw75M5f2ej7mAJx8gBMF4BBWtZ' - def _run_load(tx_left, stats): logstats.thread.start(stats) - b = bigchaindb.Bigchain() + client = temp_client() + # b = bigchaindb.Bigchain() while True: - tx = b.create_transaction(b.me, USER_PUBLIC_KEY, None, 'CREATE') - tx_signed = b.sign_transaction(tx, b.me_private) - b.write_transaction(tx_signed) + tx = client.create() stats['transactions'] += 1 From ed1e71bfecc1467b66860e27a2cbf21d90bfd3b2 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 20 Apr 2016 18:18:01 +0200 Subject: [PATCH 03/11] Add tests to ProcessGroup --- tests/test_util.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 22cc0c22..d535a505 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,3 +1,4 @@ +from unittest.mock import patch, call import pytest import queue @@ -121,3 +122,24 @@ def test_pool_raises_empty_exception_when_timeout(mock_queue): with pool() as instance: assert instance == 'hello' + +@patch('multiprocessing.Process') +def test_process_group_instantiates_and_start_processes(mock_process): + from bigchaindb.util import ProcessGroup + + def noop(): + pass + + concurrency = 10 + + pg = ProcessGroup(concurrency=concurrency, group='test_group', target=noop) + pg.start() + + mock_process.assert_has_calls([call(group='test_group', target=noop, + name=None, args=(), kwargs={}, + daemon=None) + for i in range(concurrency)], any_order=True) + + for process in pg.processes: + process.start.assert_called() + From fdce0d5876b5388b49ca60a9a098aa8f79123255 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 20 Apr 2016 23:23:08 +0200 Subject: [PATCH 04/11] Fix AttributeError in test --- tests/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index d535a505..b470053c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -141,5 +141,5 @@ def test_process_group_instantiates_and_start_processes(mock_process): for i in range(concurrency)], any_order=True) for process in pg.processes: - process.start.assert_called() + process.start.assert_called_with() From dcbaedf7ea0491a901826e1a4d8c5ad80975e186 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 21 Apr 2016 13:38:02 +0200 Subject: [PATCH 05/11] Unified place for the version number added --version to bigchain-cli --- bigchaindb/commands/utils.py | 8 +++++++- docs/source/conf.py | 7 ++++++- setup.py | 8 +++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 31d4a46d..c23b25ec 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -5,6 +5,8 @@ for ``argparse.ArgumentParser``. import argparse import multiprocessing as mp +from bigchaindb.version import __version__ + def start(parser, scope): """Utility function to execute a subcommand. @@ -46,7 +48,7 @@ def start(parser, scope): func(args) -base_parser = argparse.ArgumentParser(add_help=False) +base_parser = argparse.ArgumentParser(add_help=False, prog='bigchaindb') base_parser.add_argument('-c', '--config', help='Specify the location of the configuration file') @@ -55,3 +57,7 @@ base_parser.add_argument('-y', '--yes', '--yes-please', action='store_true', help='Assume "yes" as answer to all prompts and run ' 'non-interactively') + +base_parser.add_argument('-v', '--version', + action='version', + version='%(prog)s {}'.format(__version__)) diff --git a/docs/source/conf.py b/docs/source/conf.py index 766278db..b2dd5a79 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,6 +30,11 @@ from recommonmark.parser import CommonMarkParser # ones. import sphinx_rtd_theme +# get version +_version = {} +with open('../../bigchaindb/version.py') as fp: + exec(fp.read(), _version) + extensions = [ 'sphinx.ext.autodoc', @@ -71,7 +76,7 @@ author = 'BigchainDB Contributors' # The short X.Y version. version = '0.1' # The full version, including alpha/beta/rc tags. -release = '0.1.4' +release = _version['__version__'] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 8f4cfab6..809bcd73 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,14 @@ BigchainDB: A Scalable Blockchain Database For full docs visit https://bigchaindb.readthedocs.org """ +import os, sys from setuptools import setup, find_packages +# get the version +version = {} +with open('bigchaindb/version.py') as fp: + exec(fp.read(), version) + tests_require = [ 'pytest', 'coverage', @@ -32,7 +38,7 @@ docs_require = [ setup( name='BigchainDB', - version='0.1.4', + version=version['__version__'], description='BigchainDB: A Scalable Blockchain Database', long_description=__doc__, url='https://github.com/BigchainDB/bigchaindb/', From 7847dea1ccd1d3053d5d641fb8b71a56c144ea23 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 21 Apr 2016 13:40:57 +0200 Subject: [PATCH 06/11] added version.py --- bigchaindb/version.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 bigchaindb/version.py diff --git a/bigchaindb/version.py b/bigchaindb/version.py new file mode 100644 index 00000000..e2888cce --- /dev/null +++ b/bigchaindb/version.py @@ -0,0 +1 @@ +__version__ = '0.1.5' \ No newline at end of file From 7498fafacc32e2f4f4ae4a7335bc9fe962683f0b Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 21 Apr 2016 13:43:50 +0200 Subject: [PATCH 07/11] removed unused imports --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 809bcd73..fd63eb0c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ BigchainDB: A Scalable Blockchain Database For full docs visit https://bigchaindb.readthedocs.org """ -import os, sys from setuptools import setup, find_packages # get the version From 33825ad9dc5462dbf08a8cfdf68fe648fd3b035e Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 21 Apr 2016 14:13:41 +0200 Subject: [PATCH 08/11] added short_version field because of readthedocs --- bigchaindb/version.py | 3 ++- docs/source/conf.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index e2888cce..8902322a 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1 +1,2 @@ -__version__ = '0.1.5' \ No newline at end of file +__version__ = '0.1.5' +__short_version__ = '0.1' \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index b2dd5a79..be078160 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,7 +74,7 @@ author = 'BigchainDB Contributors' # built documents. # # The short X.Y version. -version = '0.1' +version = _version['__short_version__'] # The full version, including alpha/beta/rc tags. release = _version['__version__'] From 8ed76ca234d694d00b7aaf225539eb077545f3db Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 21 Apr 2016 14:40:51 +0200 Subject: [PATCH 09/11] Import version in bigchaindb.__init__ Print current version with import bigchaindb bigchaindb.__version__ --- bigchaindb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 3c94ca38..43278269 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -37,4 +37,4 @@ config = { # for more info. _config = copy.deepcopy(config) from bigchaindb.core import Bigchain # noqa - +from bigchaindb.version import __version__ # noqa From 970219051650a4632c0437f355f80e9a6b1523b6 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 21 Apr 2016 16:01:21 +0200 Subject: [PATCH 10/11] code cleanup --- bigchaindb/util.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index e17874ac..64e2095a 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -202,10 +202,6 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): }, } """ - current_owners = current_owners if isinstance(current_owners, list) else [current_owners] - new_owners = new_owners if isinstance(new_owners, list) else [new_owners] - inputs = inputs if isinstance(inputs, list) else [inputs] - # validate arguments (owners and inputs should be lists) if not isinstance(current_owners, list): current_owners = [current_owners] From d9da8dc17ca8203c6201e4228901715814d11552 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 21 Apr 2016 16:16:55 +0200 Subject: [PATCH 11/11] Set logging level for requests to WARNING --- bigchaindb/config_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 98be7fc9..3ec17656 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -22,6 +22,8 @@ from pkg_resources import iter_entry_points, ResolutionError import bigchaindb from bigchaindb.consensus import AbstractConsensusRules +# TODO: move this to a proper configuration file for logging +logging.getLogger('requests').setLevel(logging.WARNING) logger = logging.getLogger(__name__) CONFIG_DEFAULT_PATH = os.environ.setdefault(