From 1d4eddf635d6f23074d3be3690a74ce6dba66287 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 9 May 2016 17:27:16 +0200 Subject: [PATCH 1/9] Created templates and util functions for benchmark testing. --- benchmarking-tests/benchmark_utils.py | 26 ++++++++++++++++++ benchmarking-tests/fabfile.py | 39 +++++++++++++++++++++++++++ benchmarking-tests/hostlist.py | 8 ++++++ benchmarking-tests/test1/README.md | 11 ++++++++ 4 files changed, 84 insertions(+) create mode 100644 benchmarking-tests/benchmark_utils.py create mode 100644 benchmarking-tests/fabfile.py create mode 100644 benchmarking-tests/hostlist.py create mode 100644 benchmarking-tests/test1/README.md diff --git a/benchmarking-tests/benchmark_utils.py b/benchmarking-tests/benchmark_utils.py new file mode 100644 index 00000000..50ea586b --- /dev/null +++ b/benchmarking-tests/benchmark_utils.py @@ -0,0 +1,26 @@ +import sys +import multiprocessing as mp +import uuid +from bigchaindb import Bigchain +from bigchaindb.util import ProcessGroup + + +def create_write_transaction(tx_left): + b = Bigchain() + while tx_left > 0: + # use uuid to prevent duplicate transactions (transactions with the same hash) + tx = b.create_transaction(b.me, b.me, None, 'CREATE', payload={'msg': str(uuid.uuid4())}) + tx_signed = b.sign_transaction(tx, b.me_private) + b.write_transaction(tx_signed) + tx_left -= 1 + + +def add_to_backlog(num_transactions=10000): + tx_left = num_transactions // mp.cpu_count() + workers = ProcessGroup(target=create_write_transaction, args=(tx_left,)) + workers.start() + + +if __name__ == '__main__': + add_to_backlog(int(sys.argv[1])) + diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py new file mode 100644 index 00000000..71c682c1 --- /dev/null +++ b/benchmarking-tests/fabfile.py @@ -0,0 +1,39 @@ +from __future__ import with_statement, unicode_literals + +from fabric.api import sudo, env, hosts +from fabric.api import task, parallel +from fabric.contrib.files import sed +from fabric.operations import run, put +from fabric.context_managers import settings + +from hostlist import public_dns_names + +# Ignore known_hosts +# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts +env.disable_known_hosts = True + +# What remote servers should Fabric connect to? With what usernames? +env.user = 'ubuntu' +env.hosts = public_dns_names + +# SSH key files to try when connecting: +# http://docs.fabfile.org/en/1.10/usage/env.html#key-filename +env.key_filename = 'pem/bigchaindb.pem' + + +@task +@parallel +def prepare_test(): + put('benchmark_utils.py') + + +@task +@parallel +def prepare_backlog(num_transactions=10000): + run('python3 benchmark_utils.py {}'.format(num_transactions)) + + +@task +@parallel +def start_bigchaindb(): + run('screen -d -m bigchaindb start &', pty=False) diff --git a/benchmarking-tests/hostlist.py b/benchmarking-tests/hostlist.py new file mode 100644 index 00000000..00a48196 --- /dev/null +++ b/benchmarking-tests/hostlist.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""A list of the public DNS names of all the nodes in this +BigchainDB cluster/federation. +""" + +from __future__ import unicode_literals + +public_dns_names = ['ec2-52-58-162-146.eu-central-1.compute.amazonaws.com', 'ec2-52-58-15-239.eu-central-1.compute.amazonaws.com', 'ec2-52-58-160-205.eu-central-1.compute.amazonaws.com'] diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md new file mode 100644 index 00000000..1b186a94 --- /dev/null +++ b/benchmarking-tests/test1/README.md @@ -0,0 +1,11 @@ +# Transactions per second + +Measure how many blocks per second are created on the _bigchain_ with a pre filled backlog. + +1. Deploy an aws cluster http://bigchaindb.readthedocs.io/en/latest/deploy-on-aws.html +2. Copy `deploy-cluster-aws/hostlist.py` to `benchmarking-tests` + +```bash +fab prepare_test +fab prepare_backlog: # wait for process to finish +fab start_bigchaindb \ No newline at end of file From 6e285c95109626f28dbc676ff51c7c36e4559c90 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 10 May 2016 11:44:48 +0200 Subject: [PATCH 2/9] updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 99e9739c..b1d9188c 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ deploy-cluster-aws/conf/rethinkdb.conf deploy-cluster-aws/hostlist.py deploy-cluster-aws/confiles/ deploy-cluster-aws/client_confile +benchmarking-tests/hostlist.py From e2fd574c6e8e24b442f54e7dc878502f686ce49b Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 10 May 2016 11:47:49 +0200 Subject: [PATCH 3/9] Added cli to benchmark_utils --- benchmarking-tests/benchmark_utils.py | 43 +++++++++++++++++++++++---- benchmarking-tests/fabfile.py | 16 +++++++++- benchmarking-tests/hostlist.py | 2 +- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/benchmarking-tests/benchmark_utils.py b/benchmarking-tests/benchmark_utils.py index 50ea586b..38017e2f 100644 --- a/benchmarking-tests/benchmark_utils.py +++ b/benchmarking-tests/benchmark_utils.py @@ -1,8 +1,13 @@ -import sys import multiprocessing as mp import uuid +import json +import argparse + +from os.path import expanduser + from bigchaindb import Bigchain from bigchaindb.util import ProcessGroup +from bigchaindb.commands import utils def create_write_transaction(tx_left): @@ -15,12 +20,40 @@ def create_write_transaction(tx_left): tx_left -= 1 -def add_to_backlog(num_transactions=10000): - tx_left = num_transactions // mp.cpu_count() +def run_add_backlog(args): + tx_left = args.num_transactions // mp.cpu_count() workers = ProcessGroup(target=create_write_transaction, args=(tx_left,)) workers.start() -if __name__ == '__main__': - add_to_backlog(int(sys.argv[1])) +def run_update_statsd(args): + with open(expanduser('~') + '/.bigchaindb', 'r') as f: + conf = json.load(f) + + conf['statsd']['host'] = args.statsd_host + with open(expanduser('~') + '/.bigchaindb', 'w') as f: + json.dump(conf, f) + + +def main(): + parser = argparse.ArgumentParser(description='BigchainDB benchmarking utils') + subparsers = parser.add_subparsers(title='Commands', dest='command') + + # add transactions to backlog + backlog_parser = subparsers.add_parser('add-backlog', + help='Add transactions to the backlog') + backlog_parser.add_argument('num_transactions', metavar='num_transactions', type=int, default=0, + help='Number of transactions to add to the backlog') + + # update statsd configuration + statsd_parser = subparsers.add_parser('update-statsd', + help='Update statsd host') + statsd_parser.add_argument('statsd_host', metavar='statsd_host', default='localhost', + help='Hostname of the statsd server') + + utils.start(parser, globals()) + + +if __name__ == '__main__': + main() diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py index 71c682c1..770ed218 100644 --- a/benchmarking-tests/fabfile.py +++ b/benchmarking-tests/fabfile.py @@ -27,13 +27,27 @@ def prepare_test(): put('benchmark_utils.py') +@task +@parallel +def update_statsd_conf(statsd_host='localhost'): + run('python3 benchmark_utils.py update-statsd {}'.format(statsd_host)) + print('update configuration') + run('bigchaindb show-config') + + @task @parallel def prepare_backlog(num_transactions=10000): - run('python3 benchmark_utils.py {}'.format(num_transactions)) + run('python3 benchmark_utils.py add-backlog {}'.format(num_transactions)) @task @parallel def start_bigchaindb(): run('screen -d -m bigchaindb start &', pty=False) + + +@task +@parallel +def kill_bigchaindb(): + run('killall bigchaindb') diff --git a/benchmarking-tests/hostlist.py b/benchmarking-tests/hostlist.py index 00a48196..4d3d1641 100644 --- a/benchmarking-tests/hostlist.py +++ b/benchmarking-tests/hostlist.py @@ -5,4 +5,4 @@ BigchainDB cluster/federation. from __future__ import unicode_literals -public_dns_names = ['ec2-52-58-162-146.eu-central-1.compute.amazonaws.com', 'ec2-52-58-15-239.eu-central-1.compute.amazonaws.com', 'ec2-52-58-160-205.eu-central-1.compute.amazonaws.com'] +public_dns_names = ['ec2-52-58-110-9.eu-central-1.compute.amazonaws.com', 'ec2-52-58-154-94.eu-central-1.compute.amazonaws.com', 'ec2-52-58-166-93.eu-central-1.compute.amazonaws.com'] From 37c34f7383a6e52c386af5abb105f0a5f334fcf3 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 10 May 2016 11:48:27 +0200 Subject: [PATCH 4/9] remove tmp file --- benchmarking-tests/hostlist.py | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 benchmarking-tests/hostlist.py diff --git a/benchmarking-tests/hostlist.py b/benchmarking-tests/hostlist.py deleted file mode 100644 index 4d3d1641..00000000 --- a/benchmarking-tests/hostlist.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -"""A list of the public DNS names of all the nodes in this -BigchainDB cluster/federation. -""" - -from __future__ import unicode_literals - -public_dns_names = ['ec2-52-58-110-9.eu-central-1.compute.amazonaws.com', 'ec2-52-58-154-94.eu-central-1.compute.amazonaws.com', 'ec2-52-58-166-93.eu-central-1.compute.amazonaws.com'] From d735d677fd034549beba71f086b898a06715aa00 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 10 May 2016 12:04:32 +0200 Subject: [PATCH 5/9] updated test README --- benchmarking-tests/test1/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md index 1b186a94..4cc96d8a 100644 --- a/benchmarking-tests/test1/README.md +++ b/benchmarking-tests/test1/README.md @@ -7,5 +7,6 @@ Measure how many blocks per second are created on the _bigchain_ with a pre fill ```bash fab prepare_test +fab update_statsd_conf: fab prepare_backlog: # wait for process to finish fab start_bigchaindb \ No newline at end of file From feb402ee06f73f1e4e631877acc78fbe6c245854 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 10 May 2016 16:19:13 +0200 Subject: [PATCH 6/9] Renamed prepare_test() as put_benchmark_utils() --- benchmarking-tests/fabfile.py | 2 +- benchmarking-tests/test1/README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py index 770ed218..b6bb3c39 100644 --- a/benchmarking-tests/fabfile.py +++ b/benchmarking-tests/fabfile.py @@ -23,7 +23,7 @@ env.key_filename = 'pem/bigchaindb.pem' @task @parallel -def prepare_test(): +def put_benchmark_utils(): put('benchmark_utils.py') diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md index 4cc96d8a..dd2115ef 100644 --- a/benchmarking-tests/test1/README.md +++ b/benchmarking-tests/test1/README.md @@ -6,7 +6,8 @@ Measure how many blocks per second are created on the _bigchain_ with a pre fill 2. Copy `deploy-cluster-aws/hostlist.py` to `benchmarking-tests` ```bash -fab prepare_test +fab put_benchmark_utils fab update_statsd_conf: fab prepare_backlog: # wait for process to finish -fab start_bigchaindb \ No newline at end of file +fab start_bigchaindb +``` \ No newline at end of file From be9e44140e20001c82be3f7934de7d2969f15b87 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 10 May 2016 16:36:00 +0200 Subject: [PATCH 7/9] Renamed update-statsd command to set-statsd-host --- benchmarking-tests/benchmark_utils.py | 11 ++++++----- benchmarking-tests/fabfile.py | 4 ++-- benchmarking-tests/test1/README.md | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/benchmarking-tests/benchmark_utils.py b/benchmarking-tests/benchmark_utils.py index 38017e2f..f05ee0f9 100644 --- a/benchmarking-tests/benchmark_utils.py +++ b/benchmarking-tests/benchmark_utils.py @@ -14,7 +14,8 @@ def create_write_transaction(tx_left): b = Bigchain() while tx_left > 0: # use uuid to prevent duplicate transactions (transactions with the same hash) - tx = b.create_transaction(b.me, b.me, None, 'CREATE', payload={'msg': str(uuid.uuid4())}) + tx = b.create_transaction(b.me, b.me, None, 'CREATE', + payload={'msg': str(uuid.uuid4())}) tx_signed = b.sign_transaction(tx, b.me_private) b.write_transaction(tx_signed) tx_left -= 1 @@ -26,7 +27,7 @@ def run_add_backlog(args): workers.start() -def run_update_statsd(args): +def run_set_statsd_host(args): with open(expanduser('~') + '/.bigchaindb', 'r') as f: conf = json.load(f) @@ -45,9 +46,9 @@ def main(): backlog_parser.add_argument('num_transactions', metavar='num_transactions', type=int, default=0, help='Number of transactions to add to the backlog') - # update statsd configuration - statsd_parser = subparsers.add_parser('update-statsd', - help='Update statsd host') + # set statsd host + statsd_parser = subparsers.add_parser('set-statsd-host', + help='Set statsd host') statsd_parser.add_argument('statsd_host', metavar='statsd_host', default='localhost', help='Hostname of the statsd server') diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py index b6bb3c39..ddfb36dd 100644 --- a/benchmarking-tests/fabfile.py +++ b/benchmarking-tests/fabfile.py @@ -29,8 +29,8 @@ def put_benchmark_utils(): @task @parallel -def update_statsd_conf(statsd_host='localhost'): - run('python3 benchmark_utils.py update-statsd {}'.format(statsd_host)) +def set_statsd_host(statsd_host='localhost'): + run('python3 benchmark_utils.py set-statsd-host {}'.format(statsd_host)) print('update configuration') run('bigchaindb show-config') diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md index dd2115ef..302c9268 100644 --- a/benchmarking-tests/test1/README.md +++ b/benchmarking-tests/test1/README.md @@ -7,7 +7,7 @@ Measure how many blocks per second are created on the _bigchain_ with a pre fill ```bash fab put_benchmark_utils -fab update_statsd_conf: +fab set_statsd_host: fab prepare_backlog: # wait for process to finish fab start_bigchaindb ``` \ No newline at end of file From c4db2c5d661214f522faf4ef59b9b7e18619e9c5 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 10 May 2016 16:46:17 +0200 Subject: [PATCH 8/9] Added README for benchmarking tests --- benchmarking-tests/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 benchmarking-tests/README.md diff --git a/benchmarking-tests/README.md b/benchmarking-tests/README.md new file mode 100644 index 00000000..3ae00969 --- /dev/null +++ b/benchmarking-tests/README.md @@ -0,0 +1,3 @@ +# Benchmarking tests + +This folder contains util files and test case folders to benchmark the performance of a BigchainDB federation. \ No newline at end of file From 3636f2da9c43440424f889c9aec0544a7c397882 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 10 May 2016 17:28:03 +0200 Subject: [PATCH 9/9] Make symbolic links to hostlist.py and pem file, rather than copy --- benchmarking-tests/test1/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md index 302c9268..12d08ce2 100644 --- a/benchmarking-tests/test1/README.md +++ b/benchmarking-tests/test1/README.md @@ -3,7 +3,15 @@ Measure how many blocks per second are created on the _bigchain_ with a pre filled backlog. 1. Deploy an aws cluster http://bigchaindb.readthedocs.io/en/latest/deploy-on-aws.html -2. Copy `deploy-cluster-aws/hostlist.py` to `benchmarking-tests` +2. Make a symbolic link to hostlist.py: `ln -s ../deploy-cluster-aws/hostlist.py .` +3. Make a symbolic link to bigchaindb.pem: +```bash +mkdir pem +cd pem +ln -s ../deploy-cluster-aws/pem/bigchaindb.pem . +``` + +Then: ```bash fab put_benchmark_utils