diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index aec03314..4bb76c44 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -1,7 +1,6 @@ import os import copy -from bigchaindb.core import Bigchain # noqa def e(key, default=None, conv=None): @@ -35,10 +34,17 @@ config = { 'private': e('BIGCHAIN_KEYPAIR_PRIVATE') }, 'keyring': [ - ] + ], + 'statsd': { + 'host': e('BIGCHAIN_STATSD_HOST', default='localhost'), + 'port': e('BIGCHAIN_STATSD_PORT', default=8125), + 'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01) + } } # We need to maintain a backup copy of the original config dict in case # the user wants to reconfigure the node. Check ``bigchaindb.config_utils`` # for more info. _config = copy.deepcopy(config) +from bigchaindb.core import Bigchain # noqa + diff --git a/bigchaindb/block.py b/bigchaindb/block.py index ed1257e9..0b021969 100644 --- a/bigchaindb/block.py +++ b/bigchaindb/block.py @@ -4,11 +4,16 @@ import queue import rethinkdb as r +import bigchaindb from bigchaindb import Bigchain +from bigchaindb.monitor import Monitor + logger = logging.getLogger(__name__) +monitor = Monitor() + class Block(object): @@ -52,6 +57,7 @@ class Block(object): b = Bigchain() while True: + monitor.gauge('tx_queue_gauge', self.q_tx_to_validate.qsize(), rate=bigchaindb.config['statsd']['rate']) tx = self.q_tx_to_validate.get() # poison pill diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index bcbf1926..33d37822 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -59,6 +59,10 @@ def run_configure(args, skip_if_exists=False): val = conf['database'][key] conf['database'][key] = input('Database {}? (default `{}`): '.format(key, val)) or val + for key in ('host', 'port', 'rate'): + val = conf['statsd'][key] + conf['statsd'][key] = input('Statsd {}? (default `{}`): '.format(key, val)) or val + bigchaindb.config_utils.write_config(conf, config_path) print('Ready to go!') diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 112af0db..d69b009c 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -9,6 +9,10 @@ from bigchaindb import util from bigchaindb import config_utils from bigchaindb import exceptions from bigchaindb import crypto +from bigchaindb.monitor import Monitor + + +monitor = Monitor() class GenesisBlockAlreadyExistsError(Exception): @@ -63,6 +67,7 @@ class Bigchain(object): def reconnect(self): return r.connect(host=self.host, port=self.port, db=self.dbname) + @monitor.timer('create_transaction', rate=bigchaindb.config['statsd']['rate']) def create_transaction(self, current_owner, new_owner, tx_input, operation, payload=None): """Create a new transaction @@ -96,6 +101,7 @@ class Bigchain(object): public_key = crypto.PublicKey(public_key_base58) return public_key.verify(util.serialize(data), signature) + @monitor.timer('write_transaction', rate=bigchaindb.config['statsd']['rate']) def write_transaction(self, signed_transaction, durability='soft'): """Write the transaction to bigchain. @@ -230,6 +236,7 @@ class Bigchain(object): return owned + @monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']) def validate_transaction(self, transaction): """Validate a transaction. @@ -336,6 +343,7 @@ class Bigchain(object): return block + @monitor.timer('validate_block') # TODO: check that the votings structure is correctly constructed def validate_block(self, block): """Validate a block. @@ -379,6 +387,7 @@ class Bigchain(object): except Exception: return False + @monitor.timer('write_block') def write_block(self, block, durability='soft'): """Write a block to bigchain. diff --git a/bigchaindb/monitor.py b/bigchaindb/monitor.py new file mode 100644 index 00000000..9094fea3 --- /dev/null +++ b/bigchaindb/monitor.py @@ -0,0 +1,30 @@ +import statsd +from platform import node + +import bigchaindb +from bigchaindb import config_utils + +class Monitor(statsd.StatsClient): + """Set up statsd monitoring + + """ + def __init__(self, *args, **kwargs): + """Overrides statsd client, fixing prefix to messages and loading configuration + + Args: + *args: arguments (identical to Statsclient) + **kwargs: keyword arguments (identical to Statsclient) + """ + config_utils.autoconfigure() + + if not kwargs: + kwargs = {} + + # set prefix, parameters from configuration file + if 'prefix' not in kwargs: + kwargs['prefix'] = '{hostname}.'.format(hostname=node()) + if 'host' not in kwargs: + kwargs['host'] = bigchaindb.config['statsd']['host'] + if 'port' not in kwargs: + kwargs['port'] = bigchaindb.config['statsd']['port'] + super().__init__(*args, **kwargs) diff --git a/docker-compose-monitor.yml b/docker-compose-monitor.yml new file mode 100644 index 00000000..ee7fe3a4 --- /dev/null +++ b/docker-compose-monitor.yml @@ -0,0 +1,25 @@ +influxdb: + image: tutum/influxdb + ports: + - "8083:8083" + - "8086:8086" + expose: + - "8090" + - "8099" + environment: + PRE_CREATE_DB: "telegraf" + +grafana: + image: rhsimplex/grafana-bigchaindb-docker + tty: true + ports: + - "3000:3000" + links: + - influxdb:localhost + +statsd: + image: rhsimplex/docker-telegraf-statsd + ports: + - "8125:8125/udp" + links: + - influxdb:localhost \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index f8ef2a32..d15ff99b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -21,6 +21,7 @@ Table of Contents models json-serialization developer-interface + monitoring contributing faq release-notes diff --git a/docs/source/monitoring.md b/docs/source/monitoring.md new file mode 100644 index 00000000..436a42aa --- /dev/null +++ b/docs/source/monitoring.md @@ -0,0 +1,22 @@ +# Monitoring + +BigchainDB uses [statsd](https://github.com/etsy/statsd) for monitoring. To fully take advantage of this functionality requires some additional infrastructure: an agent to listen for metrics (e.g. [telegraf](https://github.com/influxdata/telegraf)), a time-series database (e.g. [influxdb](https://influxdata.com/time-series-platform/influxdb/)), and a frontend to display analytics (e.g. [Grafana](http://grafana.org/)). + +For ease of use, we've provided a docker compose file that sets up all these services for testing. Simply run in the BigchainDB directory: + +```text +$ docker-compose -f docker-compose-monitor.yml build +$ docker-compose -f docker-compose-monitor.yml up +``` + +and point a browser tab to `http://localhost:3000/dashboard/script/bigchaindb_dashboard.js`. Login and password are `admin` by default. If BigchainDB is running and processing transactions, you should see analytics—if not, [start BigchainDB](installing.html#run-bigchaindb) and load some test transactions: + +```text +$ bigchaindb-benchmark load +``` + +and refresh the page after a few seconds. + +If you're not interested in monitoring, don't worry: BigchainDB will function just fine without any monitoring setup. + +Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking! \ No newline at end of file diff --git a/tests/test_commands.py b/tests/test_commands.py index b3c1ae58..faa455d0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -60,6 +60,7 @@ def mock_bigchaindb_backup_config(monkeypatch): config = { 'keypair': {}, 'database': {'host': 'host', 'port': 12345, 'name': 'adbname'}, + 'statsd': {'host': 'host', 'port': 12345, 'rate': 0.1}, } monkeypatch.setattr('bigchaindb._config', config) diff --git a/tests/test_monitor.py b/tests/test_monitor.py new file mode 100644 index 00000000..a138b9b1 --- /dev/null +++ b/tests/test_monitor.py @@ -0,0 +1,14 @@ +from platform import node + + +def test_monitor_class_init_defaults(): + import bigchaindb + from bigchaindb.monitor import Monitor + monitor = Monitor() + assert monitor + assert len(monitor._addr) == 2 + # TODO get value from config + # assert monitor._addr[0] == bigchaindb.config['statsd']['host'] + assert monitor._addr[0] == '127.0.0.1' + assert monitor._addr[1] == bigchaindb.config['statsd']['port'] + assert monitor._prefix == node() + '.' diff --git a/tests/utils/test_config_utils.py b/tests/utils/test_config_utils.py index 9453482f..d57753cf 100644 --- a/tests/utils/test_config_utils.py +++ b/tests/utils/test_config_utils.py @@ -6,7 +6,7 @@ import bigchaindb from bigchaindb import exceptions -ORIGINAL_CONFIG = copy.deepcopy(bigchaindb.config) +ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config) @pytest.fixture(scope='function', autouse=True)