From ca21c7b321c67b02ff6ade6cce5b7bbee5c154ae Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Mar 2017 16:14:39 +0100 Subject: [PATCH 01/80] Remove all stuff related to 'bigchaindb load' in AWS scripts --- deploy-cluster-aws/awsdeploy.sh | 100 ++++++++---------- deploy-cluster-aws/example_deploy_conf.py | 4 - deploy-cluster-aws/fabfile.py | 15 --- deploy-cluster-aws/launch_ec2_nodes.py | 10 +- .../clusters-feds/aws-testing-cluster.md | 1 - 5 files changed, 46 insertions(+), 84 deletions(-) diff --git a/deploy-cluster-aws/awsdeploy.sh b/deploy-cluster-aws/awsdeploy.sh index 00d1f431..b733ef2d 100755 --- a/deploy-cluster-aws/awsdeploy.sh +++ b/deploy-cluster-aws/awsdeploy.sh @@ -39,7 +39,6 @@ fi echo "NUM_NODES = "$NUM_NODES echo "BRANCH = "$BRANCH -echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY echo "SSH_KEY_NAME" = $SSH_KEY_NAME echo "USE_KEYPAIRS_FILE = "$USE_KEYPAIRS_FILE echo "IMAGE_ID = "$IMAGE_ID @@ -85,7 +84,7 @@ if [[ $CONFILES_COUNT != $NUM_NODES ]]; then fi # Auto-generate the tag to apply to all nodes in the cluster -TAG="BDB-"$WHAT_TO_DEPLOY"-"`date +%m-%d@%H:%M` +TAG="BDB-Server-"`date +%m-%d@%H:%M` echo "TAG = "$TAG # Change the file permissions on the SSH private key file @@ -121,25 +120,24 @@ fab install_base_software fab get_pip3 fab upgrade_setuptools -if [ "$WHAT_TO_DEPLOY" == "servers" ]; then - # (Re)create the RethinkDB configuration file conf/rethinkdb.conf - if [ "$ENABLE_WEB_ADMIN" == "True" ]; then - if [ "$BIND_HTTP_TO_LOCALHOST" == "True" ]; then - python create_rethinkdb_conf.py --enable-web-admin --bind-http-to-localhost - else - python create_rethinkdb_conf.py --enable-web-admin - fi +# (Re)create the RethinkDB configuration file conf/rethinkdb.conf +if [ "$ENABLE_WEB_ADMIN" == "True" ]; then + if [ "$BIND_HTTP_TO_LOCALHOST" == "True" ]; then + python create_rethinkdb_conf.py --enable-web-admin --bind-http-to-localhost else - python create_rethinkdb_conf.py + python create_rethinkdb_conf.py --enable-web-admin fi - # Rollout RethinkDB and start it - fab prep_rethinkdb_storage:$USING_EBS - fab install_rethinkdb - fab configure_rethinkdb - fab delete_rethinkdb_data - fab start_rethinkdb +else + python create_rethinkdb_conf.py fi +# Rollout RethinkDB and start it +fab prep_rethinkdb_storage:$USING_EBS +fab install_rethinkdb +fab configure_rethinkdb +fab delete_rethinkdb_data +fab start_rethinkdb + # Rollout BigchainDB (but don't start it yet) if [ "$BRANCH" == "pypi" ]; then fab install_bigchaindb_from_pypi @@ -156,48 +154,40 @@ fi # Configure BigchainDB on all nodes -if [ "$WHAT_TO_DEPLOY" == "servers" ]; then - # The idea is to send a bunch of locally-created configuration - # files out to each of the instances / nodes. +# The idea is to send a bunch of locally-created configuration +# files out to each of the instances / nodes. - # Assume a set of $NUM_NODES BigchaindB config files - # already exists in the confiles directory. - # One can create a set using a command like - # ./make_confiles.sh confiles $NUM_NODES - # (We can't do that here now because this virtual environment - # is a Python 2 environment that may not even have - # bigchaindb installed, so bigchaindb configure can't be called) +# Assume a set of $NUM_NODES BigchaindB config files +# already exists in the confiles directory. +# One can create a set using a command like +# ./make_confiles.sh confiles $NUM_NODES +# (We can't do that here now because this virtual environment +# is a Python 2 environment that may not even have +# bigchaindb installed, so bigchaindb configure can't be called) - # Transform the config files in the confiles directory - # to have proper keyrings etc. - if [ "$USE_KEYPAIRS_FILE" == "True" ]; then - python clusterize_confiles.py -k confiles $NUM_NODES - else - python clusterize_confiles.py confiles $NUM_NODES - fi - - # Send one of the config files to each instance - for (( HOST=0 ; HOST<$NUM_NODES ; HOST++ )); do - CONFILE="bcdb_conf"$HOST - echo "Sending "$CONFILE - fab set_host:$HOST send_confile:$CONFILE - done - - # Initialize BigchainDB (i.e. Create the RethinkDB database, - # the tables, the indexes, and genesis glock). Note that - # this will only be sent to one of the nodes, see the - # definition of init_bigchaindb() in fabfile.py to see why. - fab init_bigchaindb - fab set_shards:$NUM_NODES - echo "To set the replication factor to 3, do: fab set_replicas:3" - echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" +# Transform the config files in the confiles directory +# to have proper keyrings etc. +if [ "$USE_KEYPAIRS_FILE" == "True" ]; then + python clusterize_confiles.py -k confiles $NUM_NODES else - # Deploying clients - fab send_client_confile:client_confile - - # Start sending load from the clients to the servers - fab start_bigchaindb_load + python clusterize_confiles.py confiles $NUM_NODES fi +# Send one of the config files to each instance +for (( HOST=0 ; HOST<$NUM_NODES ; HOST++ )); do + CONFILE="bcdb_conf"$HOST + echo "Sending "$CONFILE + fab set_host:$HOST send_confile:$CONFILE +done + +# Initialize BigchainDB (i.e. Create the RethinkDB database, +# the tables, the indexes, and genesis glock). Note that +# this will only be sent to one of the nodes, see the +# definition of init_bigchaindb() in fabfile.py to see why. +fab init_bigchaindb +fab set_shards:$NUM_NODES +echo "To set the replication factor to 3, do: fab set_replicas:3" +echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" + # cleanup rm add2known_hosts.sh diff --git a/deploy-cluster-aws/example_deploy_conf.py b/deploy-cluster-aws/example_deploy_conf.py index 623151ef..6aab8f30 100644 --- a/deploy-cluster-aws/example_deploy_conf.py +++ b/deploy-cluster-aws/example_deploy_conf.py @@ -23,10 +23,6 @@ NUM_NODES=3 # It's where to get the BigchainDB code to be deployed on the nodes BRANCH="master" -# WHAT_TO_DEPLOY is either "servers" or "clients" -# What do you want to deploy? -WHAT_TO_DEPLOY="servers" - # SSH_KEY_NAME is the name of the SSH private key file # in $HOME/.ssh/ # It is used for SSH communications with AWS instances. diff --git a/deploy-cluster-aws/fabfile.py b/deploy-cluster-aws/fabfile.py index 9ef24edd..737109f9 100644 --- a/deploy-cluster-aws/fabfile.py +++ b/deploy-cluster-aws/fabfile.py @@ -237,15 +237,6 @@ def send_confile(confile): run('bigchaindb show-config') -@task -@parallel -def send_client_confile(confile): - put(confile, 'tempfile') - run('mv tempfile ~/.bigchaindb') - print('For this node, bigchaindb show-config says:') - run('bigchaindb show-config') - - # Initialize BigchainDB # i.e. create the database, the tables, # the indexes, and the genesis block. @@ -278,12 +269,6 @@ def start_bigchaindb(): sudo('screen -d -m bigchaindb -y start &', pty=False) -@task -@parallel -def start_bigchaindb_load(): - sudo('screen -d -m bigchaindb load &', pty=False) - - # Install and run New Relic @task @parallel diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py index e02b7b62..5418069f 100644 --- a/deploy-cluster-aws/launch_ec2_nodes.py +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -26,7 +26,7 @@ import boto3 from awscommon import get_naeips -SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'SSH_KEY_NAME', +SETTINGS = ['NUM_NODES', 'BRANCH', 'SSH_KEY_NAME', 'USE_KEYPAIRS_FILE', 'IMAGE_ID', 'INSTANCE_TYPE', 'SECURITY_GROUP', 'USING_EBS', 'EBS_VOLUME_SIZE', 'EBS_OPTIMIZED', 'ENABLE_WEB_ADMIN', 'BIND_HTTP_TO_LOCALHOST'] @@ -77,9 +77,6 @@ if not isinstance(NUM_NODES, int): if not isinstance(BRANCH, str): raise SettingsTypeError('BRANCH should be a string') -if not isinstance(WHAT_TO_DEPLOY, str): - raise SettingsTypeError('WHAT_TO_DEPLOY should be a string') - if not isinstance(SSH_KEY_NAME, str): raise SettingsTypeError('SSH_KEY_NAME should be a string') @@ -117,11 +114,6 @@ if NUM_NODES > 64: 'The AWS deployment configuration file sets it to {}'. format(NUM_NODES)) -if WHAT_TO_DEPLOY not in ['servers', 'clients']: - raise ValueError('WHAT_TO_DEPLOY should be either "servers" or "clients". ' - 'The AWS deployment configuration file sets it to {}'. - format(WHAT_TO_DEPLOY)) - if SSH_KEY_NAME in ['not-set-yet', '', None]: raise ValueError('SSH_KEY_NAME should be set. ' 'The AWS deployment configuration file sets it to {}'. diff --git a/docs/server/source/clusters-feds/aws-testing-cluster.md b/docs/server/source/clusters-feds/aws-testing-cluster.md index ac1deff1..d4b4c12e 100644 --- a/docs/server/source/clusters-feds/aws-testing-cluster.md +++ b/docs/server/source/clusters-feds/aws-testing-cluster.md @@ -86,7 +86,6 @@ Step 2 is to make an AWS deployment configuration file, if necessary. There's an ```text NUM_NODES=3 BRANCH="master" -WHAT_TO_DEPLOY="servers" SSH_KEY_NAME="not-set-yet" USE_KEYPAIRS_FILE=False IMAGE_ID="ami-8504fdea" From 421b5b03b3ceca33d92d6c7133ed7345ef09c2cd Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Mar 2017 17:41:25 +0100 Subject: [PATCH 02/80] Changed 'federation' to 'cluster' or 'consortium' in docs and some code --- benchmarking-tests/README.md | 2 +- deploy-cluster-aws/launch_ec2_nodes.py | 2 +- docs/root/source/assets.rst | 2 +- docs/root/source/decentralized.md | 14 +++++++------- docs/root/source/diversity.md | 4 ++-- docs/root/source/immutable.md | 6 +++--- docs/root/source/terminology.md | 10 +++++----- .../source/cloud-deployment-templates/index.rst | 2 +- docs/server/source/clusters-feds/backup.md | 4 ++-- docs/server/source/clusters-feds/index.rst | 6 +++--- ...{set-up-a-federation.md => set-up-a-cluster.md} | 10 +++++----- docs/server/source/data-models/block-model.rst | 8 ++++---- docs/server/source/introduction.md | 4 ++-- docs/server/source/nodes/node-assumptions.md | 4 ++-- docs/server/source/nodes/setup-run-node.md | 8 ++++---- 15 files changed, 43 insertions(+), 43 deletions(-) rename docs/server/source/clusters-feds/{set-up-a-federation.md => set-up-a-cluster.md} (69%) diff --git a/benchmarking-tests/README.md b/benchmarking-tests/README.md index 3ae00969..d94ec70b 100644 --- a/benchmarking-tests/README.md +++ b/benchmarking-tests/README.md @@ -1,3 +1,3 @@ # Benchmarking tests -This folder contains util files and test case folders to benchmark the performance of a BigchainDB federation. \ No newline at end of file +This folder contains util files and test case folders to benchmark the performance of a BigchainDB cluster. \ No newline at end of file diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py index e02b7b62..3d10332b 100644 --- a/deploy-cluster-aws/launch_ec2_nodes.py +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -298,7 +298,7 @@ print('Writing hostlist.py') with open('hostlist.py', 'w') as f: f.write('# -*- coding: utf-8 -*-\n') f.write('"""A list of the public DNS names of all the nodes in this\n') - f.write('BigchainDB cluster/federation.\n') + f.write('BigchainDB cluster.\n') f.write('"""\n') f.write('\n') f.write('from __future__ import unicode_literals\n') diff --git a/docs/root/source/assets.rst b/docs/root/source/assets.rst index 50b8ad25..14982406 100644 --- a/docs/root/source/assets.rst +++ b/docs/root/source/assets.rst @@ -3,7 +3,7 @@ How BigchainDB is Good for Asset Registrations & Transfers BigchainDB can store data of any kind (within reason), but it's designed to be particularly good for storing asset registrations and transfers: -* The fundamental thing that one submits to a BigchainDB federation to be checked and stored (if valid) is a *transaction*, and there are two kinds: CREATE transactions and TRANSFER transactions. +* The fundamental thing that one sends to a BigchainDB cluster, to be checked and stored (if valid), is a *transaction*, and there are two kinds: CREATE transactions and TRANSFER transactions. * A CREATE transaction can be use to register any kind of asset (divisible or indivisible), along with arbitrary metadata. * An asset can have zero, one, or several owners. * The owners of an asset can specify (crypto-)conditions which must be satisified by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a transfer transaction. diff --git a/docs/root/source/decentralized.md b/docs/root/source/decentralized.md index 7f0b8e95..3b82ae46 100644 --- a/docs/root/source/decentralized.md +++ b/docs/root/source/decentralized.md @@ -4,18 +4,18 @@ Decentralization means that no one owns or controls everything, and there is no Ideally, each node in a BigchainDB cluster is owned and controlled by a different person or organization. Even if the cluster lives within one organization, it's still preferable to have each node controlled by a different person or subdivision. -We use the phrase "BigchainDB federation" (or just "federation") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A federation requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each federation, but it can be very decentralized (e.g. purely vote-based, where each node gets a vote, and there are no special leadership roles). +We use the phrase "BigchainDB consortium" (or just "consortium") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A consortium requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each consortium, but it can be very decentralized (e.g. purely vote-based, where each node gets a vote, and there are no special leadership roles). -The actual data is decentralized in that it doesn’t all get stored in one place. Each federation node stores the primary of one shard and replicas of some other shards. (A shard is a subset of the total set of documents.) Sharding and replication are handled by RethinkDB. +If sharding is turned on (i.e. if the number of shards is larger than one), then the actual data is decentralized in that no one node stores all the data. -Every node has its own locally-stored list of the public keys of other federation members: the so-called keyring. There's no centrally-stored or centrally-shared keyring. +Every node has its own locally-stored list of the public keys of other consortium members: the so-called keyring. There's no centrally-stored or centrally-shared keyring. -A federation can increase its decentralization (and its resilience) by increasing its jurisdictional diversity, geographic diversity, and other kinds of diversity. This idea is expanded upon in [the section on node diversity](diversity.html). +A consortium can increase its decentralization (and its resilience) by increasing its jurisdictional diversity, geographic diversity, and other kinds of diversity. This idea is expanded upon in [the section on node diversity](diversity.html). -There’s no node that has a long-term special position in the federation. All nodes run the same software and perform the same duties. +There’s no node that has a long-term special position in the cluster. All nodes run the same software and perform the same duties. -RethinkDB has an “admin” user which can’t be deleted and which can make big changes to the database, such as dropping a table. Right now, that’s a big security vulnerability, but we have plans to mitigate it by: +RethinkDB and MongoDB have an “admin” user which can’t be deleted and which can make big changes to the database, such as dropping a table. Right now, that’s a big security vulnerability, but we have plans to mitigate it by: 1. Locking down the admin user as much as possible. -2. Having all nodes inspect RethinkDB admin-type requests before acting on them. Requests can be checked against an evolving whitelist of allowed actions (voted on by federation nodes). +2. Having all nodes inspect admin-type requests before acting on them. Requests can be checked against an evolving whitelist of allowed actions. Nodes requesing non-allowed requests can be removed from the list of cluster nodes. It’s worth noting that the RethinkDB admin user can’t transfer assets, even today. The only way to create a valid transfer transaction is to fulfill the current (crypto) conditions on the asset, and the admin user can’t do that because the admin user doesn’t have the necessary private keys (or preimages, in the case of hashlock conditions). They’re not stored in the database. diff --git a/docs/root/source/diversity.md b/docs/root/source/diversity.md index 4819a0af..20c9afb5 100644 --- a/docs/root/source/diversity.md +++ b/docs/root/source/diversity.md @@ -6,6 +6,6 @@ Steps should be taken to make it difficult for any one actor or event to control 2. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems. 3. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes. 4. **Operating system diversity.** The servers should use a variety of operating systems, so that a security bug in one OS can’t be used to exploit enough of the nodes. -5. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a federation. For example, it provides the federation with a source of various ideas for addressing challenges. +5. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a consortium. For example, it provides the consortium with a source of various ideas for addressing challenges. -Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a federation could also have a diversity of server implementations. +Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a consortium could also have a diversity of server implementations. diff --git a/docs/root/source/immutable.md b/docs/root/source/immutable.md index 28fb5999..a20c40b8 100644 --- a/docs/root/source/immutable.md +++ b/docs/root/source/immutable.md @@ -8,12 +8,12 @@ It’s true that blockchain data is more difficult to change than usual: it’s BigchainDB achieves strong tamper-resistance in the following ways: -1. **Replication.** All data is sharded and shards are replicated in several (different) places. The replication factor can be set by the federation. The higher the replication factor, the more difficult it becomes to change or delete all replicas. +1. **Replication.** All data is sharded and shards are replicated in several (different) places. The replication factor can be set by the consortium. The higher the replication factor, the more difficult it becomes to change or delete all replicas. 2. **Internal watchdogs.** All nodes monitor all changes and if some unallowed change happens, then appropriate action is taken. For example, if a valid block is deleted, then it is put back. -3. **External watchdogs.** Federations may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For federations with publicly-readable data, the public can act as an auditor. +3. **External watchdogs.** A consortium may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For a consortium with publicly-readable data, the public can act as an auditor. 4. **Cryptographic signatures** are used throughout BigchainDB as a way to check if messages (transactions, blocks and votes) have been tampered with enroute, and as a way to verify who signed the messages. Each block is signed by the node that created it. Each vote is signed by the node that cast it. A creation transaction is signed by the node that created it, although there are plans to improve that by adding signatures from the sending client and multiple nodes; see [Issue #347](https://github.com/bigchaindb/bigchaindb/issues/347). Transfer transactions can contain multiple inputs (fulfillments, one per asset transferred). Each fulfillment will typically contain one or more signatures from the owners (i.e. the owners before the transfer). Hashlock fulfillments are an exception; there’s an open issue ([#339](https://github.com/bigchaindb/bigchaindb/issues/339)) to address that. 5. **Full or partial backups** of the database may be recorded from time to time, possibly on magnetic tape storage, other blockchains, printouts, etc. 6. **Strong security.** Node owners can adopt and enforce strong security policies. 7. **Node diversity.** Diversity makes it so that no one thing (e.g. natural disaster or operating system bug) can compromise enough of the nodes. See [the section on the kinds of node diversity](diversity.html). -Some of these things come "for free" as part of the BigchainDB software, and others require some extra effort from the federation and node owners. +Some of these things come "for free" as part of the BigchainDB software, and others require some extra effort from the consortium and node owners. diff --git a/docs/root/source/terminology.md b/docs/root/source/terminology.md index fb2a3bdf..c4ef84c2 100644 --- a/docs/root/source/terminology.md +++ b/docs/root/source/terminology.md @@ -1,6 +1,6 @@ # Terminology -There is some specialized terminology associated with BigchainDB. To get started, you should at least know what what we mean by a BigchainDB *node*, *cluster* and *federation*. +There is some specialized terminology associated with BigchainDB. To get started, you should at least know what what we mean by a BigchainDB *node*, *cluster* and *consortium*. ## Node @@ -13,10 +13,10 @@ A **BigchainDB node** is a machine or set of closely-linked machines running Ret A set of BigchainDB nodes can connect to each other to form a **cluster**. Each node in the cluster runs the same software. A cluster contains one logical RethinkDB datastore. A cluster may have additional machines to do things such as cluster monitoring. -## Federation +## Consortium -The people and organizations that run the nodes in a cluster belong to a **federation** (i.e. another organization). A federation must have some sort of governance structure to make decisions. If a cluster is run by a single company, then the federation is just that company. +The people and organizations that run the nodes in a cluster belong to a **consortium** (i.e. another organization). A consortium must have some sort of governance structure to make decisions. If a cluster is run by a single company, then the "consortium" is just that company. -**What's the Difference Between a Cluster and a Federation?** +**What's the Difference Between a Cluster and a Consortium?** -A cluster is just a bunch of connected nodes. A federation is an organization which has a cluster, and where each node in the cluster has a different operator. Confusingly, we sometimes call a federation's cluster its "federation." You can probably tell what we mean from context. \ No newline at end of file +A cluster is just a bunch of connected nodes. A consortium is an organization which has a cluster, and where each node in the cluster has a different operator. \ No newline at end of file diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 67a2ace4..666e2327 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -5,7 +5,7 @@ We have some "templates" to deploy a basic, working, but bare-bones BigchainDB n You don't have to use the tools we use in the templates. You can use whatever tools you prefer. -If you find the cloud deployment templates for nodes helpful, then you may also be interested in our scripts for :doc:`deploying a testing cluster on AWS <../clusters-feds/aws-testing-cluster>` (documented in the Clusters & Federations section). +If you find the cloud deployment templates for nodes helpful, then you may also be interested in our scripts for :doc:`deploying a testing cluster on AWS <../clusters-feds/aws-testing-cluster>` (documented in the Clusters section). .. toctree:: :maxdepth: 1 diff --git a/docs/server/source/clusters-feds/backup.md b/docs/server/source/clusters-feds/backup.md index 93fd9aac..5faf3465 100644 --- a/docs/server/source/clusters-feds/backup.md +++ b/docs/server/source/clusters-feds/backup.md @@ -64,7 +64,7 @@ In the future, it will be possible for clients to query for the blocks containin **How could we be sure blocks and votes from a client are valid?** -All blocks and votes are signed by federation nodes. Only federation nodes can produce valid signatures because only federation nodes have the necessary private keys. A client can't produce a valid signature for a block or vote. +All blocks and votes are signed by cluster nodes (owned and operated by consortium members). Only cluster nodes can produce valid signatures because only cluster nodes have the necessary private keys. A client can't produce a valid signature for a block or vote. **Could we restore an entire BigchainDB database using client-saved blocks and votes?** @@ -109,7 +109,7 @@ Considerations for BigchainDB: Although it's not advertised as such, RethinkDB's built-in replication feature is similar to continous backup, except the "backup" (i.e. the set of replica shards) is spread across all the nodes. One could take that idea a bit farther by creating a set of backup-only servers with one full backup: * Give all the original BigchainDB nodes (RethinkDB nodes) the server tag `original`. This is the default if you used the RethinkDB config file suggested in the section titled [Configure RethinkDB Server](../dev-and-test/setup-run-node.html#configure-rethinkdb-server). -* Set up a group of servers running RethinkDB only, and give them the server tag `backup`. The `backup` servers could be geographically separated from all the `original` nodes (or not; it's up to the federation). +* Set up a group of servers running RethinkDB only, and give them the server tag `backup`. The `backup` servers could be geographically separated from all the `original` nodes (or not; it's up to the consortium to decide). * Clients shouldn't be able to read from or write to servers in the `backup` set. * Send a RethinkDB reconfigure command to the RethinkDB cluster to make it so that the `original` set has the same number of replicas as before (or maybe one less), and the `backup` set has one replica. Also, make sure the `primary_replica_tag='original'` so that all primary shards live on the `original` nodes. diff --git a/docs/server/source/clusters-feds/index.rst b/docs/server/source/clusters-feds/index.rst index d13221ce..93258057 100644 --- a/docs/server/source/clusters-feds/index.rst +++ b/docs/server/source/clusters-feds/index.rst @@ -1,10 +1,10 @@ -Clusters & Federations -====================== +Clusters +======== .. toctree:: :maxdepth: 1 - set-up-a-federation + set-up-a-cluster backup aws-testing-cluster diff --git a/docs/server/source/clusters-feds/set-up-a-federation.md b/docs/server/source/clusters-feds/set-up-a-cluster.md similarity index 69% rename from docs/server/source/clusters-feds/set-up-a-federation.md rename to docs/server/source/clusters-feds/set-up-a-cluster.md index ed1ddd1a..c8193dd2 100644 --- a/docs/server/source/clusters-feds/set-up-a-federation.md +++ b/docs/server/source/clusters-feds/set-up-a-cluster.md @@ -1,11 +1,11 @@ -# Set Up a Federation +# Set Up a Cluster -This section is about how to set up a BigchainDB _federation_, where each node is operated by a different operator. If you want to set up and run a testing cluster on AWS (where all nodes are operated by you), then see [the section about that](aws-testing-cluster.html). +This section is about how to set up a BigchainDB cluster where each node is operated by a different operator. If you want to set up and run a testing cluster on AWS (where all nodes are operated by you), then see [the section about that](aws-testing-cluster.html). ## Initial Checklist -* Do you have a governance process for making federation-level decisions, such as how to admit new members? +* Do you have a governance process for making consortium-level decisions, such as how to admit new members? * What will you store in creation transactions (data payload)? Is there a data schema? * Will you use transfer transactions? Will they include a non-empty data payload? * Who will be allowed to submit transactions? Who will be allowed to read or query transactions? How will you enforce the access rules? @@ -13,7 +13,7 @@ This section is about how to set up a BigchainDB _federation_, where each node i ## Set Up the Initial Cluster -The federation must decide some things before setting up the initial cluster (initial set of BigchainDB nodes): +The consortium must decide some things before setting up the initial cluster (initial set of BigchainDB nodes): 1. Who will operate a node in the initial cluster? 2. What will the replication factor be? (It must be 3 or more for [RethinkDB failover](https://rethinkdb.com/docs/failover/) to work.) @@ -21,7 +21,7 @@ The federation must decide some things before setting up the initial cluster (in Once those things have been decided, each node operator can begin setting up their BigchainDB (production) node. -Each node operator will eventually need two pieces of information from all other nodes in the federation: +Each node operator will eventually need two pieces of information from all other nodes: 1. Their RethinkDB hostname, e.g. `rdb.farm2.organization.org` 2. Their BigchainDB public key, e.g. `Eky3nkbxDTMgkmiJC8i5hKyVFiAQNmPP4a2G4JdDxJCK` diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index 3c94fca1..8b184261 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -11,7 +11,7 @@ A block has the following structure: "timestamp": "", "transactions": [""], "node_pubkey": "", - "voters": [""] + "voters": [""] }, "signature": "" } @@ -23,9 +23,9 @@ A block has the following structure: - ``timestamp``: The Unix time when the block was created. It's provided by the node that created the block. - ``transactions``: A list of the transactions included in the block. - ``node_pubkey``: The public key of the node that created the block. - - ``voters``: A list of the public keys of federation nodes at the time the block was created. - It's the list of federation nodes which can cast a vote on this block. - This list can change from block to block, as nodes join and leave the federation. + - ``voters``: A list of the public keys of all cluster nodes at the time the block was created. + It's the list of nodes which can cast a vote on this block. + This list can change from block to block, as nodes join and leave the cluster. - ``signature``: :ref:`Cryptographic signature ` of the block by the node that created the block (i.e. the node with public key ``node_pubkey``). To generate the signature, the node signs the serialized inner ``block`` (the same thing that was hashed to determine the ``id``) using the private key corresponding to ``node_pubkey``. diff --git a/docs/server/source/introduction.md b/docs/server/source/introduction.md index b9e6bf0a..02cf5ecf 100644 --- a/docs/server/source/introduction.md +++ b/docs/server/source/introduction.md @@ -10,7 +10,7 @@ Note that there are a few kinds of nodes: - A **bare-bones node** is a node deployed in the cloud, either as part of a testing cluster or as a starting point before upgrading the node to be production-ready. Our cloud deployment templates deploy a bare-bones node, as do our scripts for deploying a testing cluster on AWS. -- A **production node** is a node that is part of a federation's BigchainDB cluster. A production node has the most components and requirements. +- A **production node** is a node that is part of a consortium's BigchainDB cluster. A production node has the most components and requirements. ## Setup Instructions for Various Cases @@ -19,7 +19,7 @@ Note that there are a few kinds of nodes: * [Set up and run a bare-bones node in the cloud](cloud-deployment-templates/index.html) * [Set up and run a local dev/test node for developing and testing BigchainDB Server](dev-and-test/setup-run-node.html) * [Deploy a testing cluster on AWS](clusters-feds/aws-testing-cluster.html) -* [Set up and run a federation (including production nodes)](clusters-feds/set-up-a-federation.html) +* [Set up and run a cluster (including production nodes)](clusters-feds/set-up-a-cluster.html) Instructions for setting up a client will be provided once there's a public test net. diff --git a/docs/server/source/nodes/node-assumptions.md b/docs/server/source/nodes/node-assumptions.md index f7e8379f..8275be32 100644 --- a/docs/server/source/nodes/node-assumptions.md +++ b/docs/server/source/nodes/node-assumptions.md @@ -1,12 +1,12 @@ # Production Node Assumptions -If you're not sure what we mean by a BigchainDB *node*, *cluster*, *federation*, or *production node*, then see [the section in the Introduction where we defined those terms](../introduction.html#some-basic-vocabulary). +If you're not sure what we mean by a BigchainDB *node*, *cluster*, *consortium*, or *production node*, then see [the section in the Introduction where we defined those terms](../introduction.html#some-basic-vocabulary). We make some assumptions about production nodes: 1. **Each production node is set up and managed by an experienced professional system administrator (or a team of them).** -2. Each production node in a federation's cluster is managed by a different person or team. +2. Each production node in a cluster is managed by a different person or team. Because of the first assumption, we don't provide a detailed cookbook explaining how to secure a server, or other things that a sysadmin should know. (We do provide some [templates](../cloud-deployment-templates/index.html), but those are just a starting point.) diff --git a/docs/server/source/nodes/setup-run-node.md b/docs/server/source/nodes/setup-run-node.md index 41a9cdd1..cace5003 100644 --- a/docs/server/source/nodes/setup-run-node.md +++ b/docs/server/source/nodes/setup-run-node.md @@ -19,7 +19,7 @@ There are some [notes on BigchainDB-specific firewall setup](../appendices/firew A BigchainDB node uses its system clock to generate timestamps for blocks and votes, so that clock should be kept in sync with some standard clock(s). The standard way to do that is to run an NTP daemon (Network Time Protocol daemon) on the node. (You could also use tlsdate, which uses TLS timestamps rather than NTP, but don't: it's not very accurate and it will break with TLS 1.3, which removes the timestamp.) -NTP is a standard protocol. There are many NTP daemons implementing it. We don't recommend a particular one. On the contrary, we recommend that different nodes in a federation run different NTP daemons, so that a problem with one daemon won't affect all nodes. +NTP is a standard protocol. There are many NTP daemons implementing it. We don't recommend a particular one. On the contrary, we recommend that different nodes in a cluster run different NTP daemons, so that a problem with one daemon won't affect all nodes. Please see the [notes on NTP daemon setup](../appendices/ntp-notes.html) in the Appendices. @@ -72,7 +72,7 @@ direct-io join=node0_hostname:29015 join=node1_hostname:29015 join=node2_hostname:29015 -# continue until there's a join= line for each node in the federation +# continue until there's a join= line for each node in the cluster ``` * `directory=/data` tells the RethinkDB node to store its share of the database data in `/data`. @@ -153,7 +153,7 @@ Edit the created config file: * Open `$HOME/.bigchaindb` (the created config file) in your text editor. * Change `"server": {"bind": "localhost:9984", ... }` to `"server": {"bind": "0.0.0.0:9984", ... }`. This makes it so traffic can come from any IP address to port 9984 (the HTTP Client-Server API port). -* Change `"keyring": []` to `"keyring": ["public_key_of_other_node_A", "public_key_of_other_node_B", "..."]` i.e. a list of the public keys of all the other nodes in the federation. The keyring should _not_ include your node's public key. +* Change `"keyring": []` to `"keyring": ["public_key_of_other_node_A", "public_key_of_other_node_B", "..."]` i.e. a list of the public keys of all the other nodes in the cluster. The keyring should _not_ include your node's public key. For more information about the BigchainDB config file, see [Configuring a BigchainDB Node](configuration.html). @@ -185,7 +185,7 @@ where: * `bigchaindb init` creates the database within RethinkDB, the tables, the indexes, and the genesis block. * `numshards` should be set to the number of nodes in the initial cluster. -* `numreplicas` should be set to the database replication factor decided by the federation. It must be 3 or more for [RethinkDB failover](https://rethinkdb.com/docs/failover/) to work. +* `numreplicas` should be set to the database replication factor decided by the consortium. It must be 3 or more for [RethinkDB failover](https://rethinkdb.com/docs/failover/) to work. Once the RethinkDB database is configured, every node operator can start BigchainDB using: ```text From 696dbe7844df014e0d308da613534b4290cd7e20 Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Tue, 14 Mar 2017 14:23:30 +0100 Subject: [PATCH 03/80] SSL connection support --- bigchaindb/backend/connection.py | 6 ++++-- bigchaindb/backend/mongodb/connection.py | 14 +++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index c1f0a629..cf6bece7 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None): + connection_timeout=None, replicaset=None, ssl=False): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -50,6 +50,8 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, # to handle these these additional args. In case of RethinkDBConnection # it just does not do anything with it. replicaset = replicaset or bigchaindb.config['database'].get('replicaset') + ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ + else ssl try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -63,7 +65,7 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, logger.debug('Connection: {}'.format(Class)) return Class(host=host, port=port, dbname=dbname, max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset) + replicaset=replicaset, ssl=ssl) class Connection: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 8688e243..274d64c1 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, **kwargs): + def __init__(self, replicaset=None, ssl=False, **kwargs): """Create a new Connection instance. Args: @@ -28,6 +28,8 @@ class MongoDBConnection(Connection): super().__init__(**kwargs) self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] + self.ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ + else ssl @property def db(self): @@ -71,14 +73,15 @@ class MongoDBConnection(Connection): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout) + initialize_replica_set(self.host, self.port, self.connection_timeout, self.ssl) # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. return pymongo.MongoClient(self.host, self.port, replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout) + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) # `initialize_replica_set` might raise `ConnectionFailure` or `OperationFailure`. except (pymongo.errors.ConnectionFailure, @@ -86,7 +89,7 @@ class MongoDBConnection(Connection): raise ConnectionError() from exc -def initialize_replica_set(host, port, connection_timeout): +def initialize_replica_set(host, port, connection_timeout, ssl): """Initialize a replica set. If already initialized skip.""" # Setup a MongoDB connection @@ -95,7 +98,8 @@ def initialize_replica_set(host, port, connection_timeout): # you try to connect to a replica set that is not yet initialized conn = pymongo.MongoClient(host=host, port=port, - serverselectiontimeoutms=connection_timeout) + serverselectiontimeoutms=connection_timeout, + ssl=ssl) _check_replica_set(conn) host = '{}:{}'.format(bigchaindb.config['database']['host'], bigchaindb.config['database']['port']) From 4daeff28f82c95096eefdf1ffe2e5c21034ce130 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 14 Mar 2017 20:23:46 +0100 Subject: [PATCH 04/80] Tip for `az acs kubernetes get-credentials...` Added a tip for when `$ az acs kubernetes get-credentials...` command gives an error after you enter the correct password. --- .../cloud-deployment-templates/node-on-kubernetes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst index e1ed43e7..199694d4 100644 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -31,6 +31,12 @@ then you can get the ``~/.kube/config`` file using: --resource-group \ --name +If it asks for a password (to unlock the SSH key) +and you enter the correct password, +but you get an error message, +then try adding ``--ssh-key-file ~/.ssh/`` +to the above command (i.e. the path to the private key). + Step 3: Create Storage Classes ------------------------------ From ea6ce5c1a1cf013675ed0d4dd109084aac341649 Mon Sep 17 00:00:00 2001 From: Krish Date: Wed, 15 Mar 2017 16:22:49 +0100 Subject: [PATCH 05/80] Single node/cluster bootstrap and node addition workflow in k8s (#1278) * Combining configs * Combining the persistent volume claims into a single file. * Combining the storage classes into a single file. * Updating documentation * Multiple changes * Support for ConfigMap * Custom MongoDB container for BigchainDB * Update documentation to run a single node on k8s * Additional documentation * Documentation to add a node to an existing BigchainDB cluster * Commit on rolling upgrades * Fixing minor documentation mistakes * Documentation updates as per @ttmc's comments * Block formatting error * Change in ConfigMap yaml config --- docs/root/source/terminology.md | 4 +- .../add-node-on-kubernetes.rst | 168 +++++++++++++ .../cloud-deployment-templates/index.rst | 2 +- .../node-on-kubernetes.rst | 221 +++++++++++++++--- k8s/deprecated.to.del/mongo-statefulset.yaml | 57 +++++ k8s/mongodb/container/Dockerfile | 12 + k8s/mongodb/container/Makefile | 51 ++++ k8s/mongodb/container/README.md | 88 +++++++ k8s/mongodb/container/mongod.conf.template | 89 +++++++ .../mongod_entrypoint/mongod_entrypoint.go | 154 ++++++++++++ k8s/mongodb/mongo-cm.yaml | 13 ++ k8s/mongodb/mongo-data-configdb-pvc.yaml | 18 -- k8s/mongodb/mongo-data-configdb-sc.yaml | 12 - k8s/mongodb/mongo-data-db-pvc.yaml | 18 -- k8s/mongodb/mongo-data-db-sc.yaml | 12 - k8s/mongodb/mongo-pvc.yaml | 35 +++ k8s/mongodb/mongo-sc.yaml | 23 ++ k8s/mongodb/mongo-ss.yaml | 22 +- 18 files changed, 900 insertions(+), 99 deletions(-) create mode 100644 docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst create mode 100644 k8s/deprecated.to.del/mongo-statefulset.yaml create mode 100644 k8s/mongodb/container/Dockerfile create mode 100644 k8s/mongodb/container/Makefile create mode 100644 k8s/mongodb/container/README.md create mode 100644 k8s/mongodb/container/mongod.conf.template create mode 100644 k8s/mongodb/container/mongod_entrypoint/mongod_entrypoint.go create mode 100644 k8s/mongodb/mongo-cm.yaml delete mode 100644 k8s/mongodb/mongo-data-configdb-pvc.yaml delete mode 100644 k8s/mongodb/mongo-data-configdb-sc.yaml delete mode 100644 k8s/mongodb/mongo-data-db-pvc.yaml delete mode 100644 k8s/mongodb/mongo-data-db-sc.yaml create mode 100644 k8s/mongodb/mongo-pvc.yaml create mode 100644 k8s/mongodb/mongo-sc.yaml diff --git a/docs/root/source/terminology.md b/docs/root/source/terminology.md index fb2a3bdf..25fd00f6 100644 --- a/docs/root/source/terminology.md +++ b/docs/root/source/terminology.md @@ -5,7 +5,7 @@ There is some specialized terminology associated with BigchainDB. To get started ## Node -A **BigchainDB node** is a machine or set of closely-linked machines running RethinkDB Server, BigchainDB Server, and related software. (A "machine" might be a bare-metal server, a virtual machine or a container.) Each node is controlled by one person or organization. +A **BigchainDB node** is a machine or set of closely-linked machines running RethinkDB/MongoDB Server, BigchainDB Server, and related software. (A "machine" might be a bare-metal server, a virtual machine or a container.) Each node is controlled by one person or organization. ## Cluster @@ -19,4 +19,4 @@ The people and organizations that run the nodes in a cluster belong to a **feder **What's the Difference Between a Cluster and a Federation?** -A cluster is just a bunch of connected nodes. A federation is an organization which has a cluster, and where each node in the cluster has a different operator. Confusingly, we sometimes call a federation's cluster its "federation." You can probably tell what we mean from context. \ No newline at end of file +A cluster is just a bunch of connected nodes. A federation is an organization which has a cluster, and where each node in the cluster has a different operator. Confusingly, we sometimes call a federation's cluster its "federation." You can probably tell what we mean from context. diff --git a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst new file mode 100644 index 00000000..542d3d2b --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst @@ -0,0 +1,168 @@ +Add a BigchainDB Node in a Kubernetes Cluster +============================================= + +**Refer this document if you want to add a new BigchainDB node to an existing +cluster** + +**If you want to start your first BigchainDB node in the BigchainDB cluster, +refer** +:doc:`this ` + + +Terminology Used +---------------- + +``existing cluster`` will refer to the existing (or any one of the existing) +Kubernetes cluster that already hosts a BigchainDB instance with a MongoDB +backend. + +``ctx-1`` will refer to the kubectl context of the existing cluster. + +``new cluster`` will refer to the new Kubernetes cluster that will run a new +BigchainDB instance with a MongoDB backend. + +``ctx-2`` will refer to the kubectl context of the new cluster. + +``new MongoDB instance`` will refer to the MongoDB instance in the new cluster. + +``existing MongoDB instance`` will refer to the MongoDB instance in the +existing cluster. + +``new BigchainDB instance`` will refer to the BigchainDB instance in the new +cluster. + +``existing BigchainDB instance`` will refer to the BigchainDB instance in the +existing cluster. + + +Step 1: Prerequisites +--------------------- + +* You will need to have a public and private key for the new BigchainDB + instance you will set up. + +* The public key should be shared offline with the other existing BigchainDB + instances. The means to achieve this requirement is beyond the scope of this + document. + +* You will need the public keys of all the existing BigchainDB instances. The + means to achieve this requirement is beyond the scope of this document. + +* A new Kubernetes cluster setup with kubectl configured to access it. + If you are using Kubernetes on Azure Container Server (ACS), please refer + our documentation `here ` for the set up. + +If you haven't read our guide to set up a +:doc:`node on Kubernetes `, now is a good time to jump in +there and then come back here as these instructions build up from there. + + +NOTE: If you are managing multiple kubernetes clusters, from your local +system, you can run ``kubectl config view`` to list all the contexts that +are available for the local kubectl. +To target a specific cluster, add a ``--context`` flag to the kubectl CLI. For +example: + +.. code:: bash + + $ kubectl --context ctx-1 apply -f example.yaml + $ kubectl --context ctx-2 apply -f example.yaml + $ kubectl --context ctx-1 proxy --port 8001 + $ kubectl --context ctx-2 proxy --port 8002 + + +Step 2: Prepare the New Kubernetes cluster +------------------------------------------ +Follow the steps in the sections to set up Storage Classes and Persisten Volume +Claims, and to run MongoDB in the new cluster: + +1. :ref:`Add Storage Classes ` +2. :ref:`Add Persistent Volume Claims ` +3. :ref:`Create the Config Map ` +4. :ref:`Run MongoDB instance ` + + +Step 3: Add the New MongoDB Instance to the Existing Replica Set +---------------------------------------------------------------- +Note that by ``replica set`` we are referring to the MongoDB replica set, and not +to Kubernetes' ``ReplicaSet``. + +If you are not the administrator of an existing MongoDB/BigchainDB instance, you +will have to coordinate offline with an existing administrator so that s/he can +add the new MongoDB instance to the replica set. The means to achieve this is +beyond the scope of this document. + +Add the new instance of MongoDB from an existing instance by accessing the +``mongo`` shell. + +.. code:: bash + + $ kubectl --context ctx-1 exec -it mdb-0 -c mongodb -- /bin/bash + root@mdb-0# mongo --port 27017 + +We can only add members to a replica set from the ``PRIMARY`` instance. +The ``mongo`` shell prompt should state that this is the primary member in the +replica set. +If not, then you can use the ``rs.status()`` command to find out who the +primary is and login to the ``mongo`` shell in the primary. + +Run the ``rs.add()`` command with the FQDN and port number of the other instances: + +.. code:: bash + + PRIMARY> rs.add(":") + + +Step 4: Verify the replica set membership +----------------------------------------- + +You can use the ``rs.conf()`` and the ``rs.status()`` commands available in the +mongo shell to verify the replica set membership. + +The new MongoDB instance should be listed in the membership information +displayed. + + +Step 5: Start the new BigchainDB instance +----------------------------------------- + +Get the file ``bigchaindb-dep.yaml`` from GitHub using: + +.. code:: bash + + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/bigchaindb/bigchaindb-dep.yaml + +Note that we set the ``BIGCHAINDB_DATABASE_HOST`` to ``mdb`` which is the name +of the MongoDB service defined earlier. + +Edit the ``BIGCHAINDB_KEYPAIR_PUBLIC`` with the public key of this instance, +the ``BIGCHAINDB_KEYPAIR_PRIVATE`` with the private key of this instance and +the ``BIGCHAINDB_KEYRING`` with a ``:`` delimited list of all the public keys +in the BigchainDB cluster. + +Create the required Deployment using: + +.. code:: bash + + $ kubectl --context ctx-2 apply -f bigchaindb-dep.yaml + +You can check its status using the command ``kubectl get deploy -w`` + + +Step 6: Restart the existing BigchainDB instance(s) +--------------------------------------------------- +Add public key of the new BigchainDB instance to the keyring of all the +existing instances and update the BigchainDB instances using: + +.. code:: bash + + $ kubectl --context ctx-1 replace -f bigchaindb-dep.yaml + +This will create a ``rolling deployment`` in Kubernetes where a new instance of +BigchainDB will be created, and if the health check on the new instance is +successful, the earlier one will be terminated. This ensures that there is +zero downtime during updates. + +You can login to an existing BigchainDB instance and run the ``bigchaindb +show-config`` command to see the configuration update to the keyring. + diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 67a2ace4..dee7cd4b 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -15,4 +15,4 @@ If you find the cloud deployment templates for nodes helpful, then you may also azure-quickstart-template template-kubernetes-azure node-on-kubernetes - \ No newline at end of file + add-node-on-kubernetes diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst index 199694d4..650d2f45 100644 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -1,6 +1,12 @@ -Run a BigchainDB Node in a Kubernetes Cluster -============================================= +Bootstrap a BigchainDB Node in a Kubernetes Cluster +=================================================== +**Refer this document if you are starting your first BigchainDB instance in +a BigchainDB cluster or starting a stand-alone BigchainDB instance** + +**If you want to add a new BigchainDB node to an existing cluster, refer** +:doc:`this ` + Assuming you already have a `Kubernetes `_ cluster up and running, this page describes how to run a BigchainDB node in it. @@ -90,24 +96,21 @@ For future reference, the command to create a storage account is `az storage account create `_. -Get the files ``mongo-data-db-sc.yaml`` and ``mongo-data-configdb-sc.yaml`` -from GitHub using: +Get the file ``mongo-sc.yaml`` from GitHub using: .. code:: bash - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-data-db-sc.yaml - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-data-configdb-sc.yaml + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-sc.yaml You may want to update the ``parameters.location`` field in both the files to specify the location you are using in Azure. -Create the required StorageClass using +Create the required storage classes using .. code:: bash - $ kubectl apply -f mongo-data-db-sc.yaml - $ kubectl apply -f mongo-data-configdb-sc.yaml + $ kubectl apply -f mongo-sc.yaml You can check if it worked using ``kubectl get storageclasses``. @@ -128,13 +131,11 @@ Step 4: Create Persistent Volume Claims Next, we'll create two PersistentVolumeClaim objects ``mongo-db-claim`` and ``mongo-configdb-claim``. -Get the files ``mongo-data-db-sc.yaml`` and ``mongo-data-configdb-sc.yaml`` -from GitHub using: +Get the file ``mongo-pvc.yaml`` from GitHub using: .. code:: bash - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-data-db-pvc.yaml - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-data-configdb-pvc.yaml + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-pvc.yaml Note how there's no explicit mention of Azure, AWS or whatever. ``ReadWriteOnce`` (RWO) means the volume can be mounted as @@ -147,12 +148,11 @@ by AzureDisk.) You may want to update the ``spec.resources.requests.storage`` field in both the files to specify a different disk size. -Create the required PersistentVolumeClaim using: +Create the required Persistent Volume Claims using: .. code:: bash - $ kubectl apply -f mongo-data-db-pvc.yaml - $ kubectl apply -f mongo-data-configdb-pvc.yaml + $ kubectl apply -f mongo-pvc.yaml You can check its status using: ``kubectl get pvc -w`` @@ -161,9 +161,81 @@ Initially, the status of persistent volume claims might be "Pending" but it should become "Bound" fairly quickly. +Step 5: Create the Config Map - Optional +---------------------------------------- + +This step is required only if you are planning to set up multiple +`BigchainDB nodes +`_, else you can +skip to the :ref:`next step `. + +MongoDB reads the local ``/etc/hosts`` file while bootstrapping a replica set +to resolve the hostname provided to the ``rs.initiate()`` command. It needs to +ensure that the replica set is being initialized in the same instance where +the MongoDB instance is running. + +To achieve this, we create a ConfigMap with the FQDN of the MongoDB instance +and populate the ``/etc/hosts`` file with this value so that a replica set can +be created seamlessly. + +Get the file ``mongo-cm.yaml`` from GitHub using: + +.. code:: bash + + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-cm.yaml + +You may want to update the ``data.fqdn`` field in the file before creating the +ConfigMap. ``data.fqdn`` field will be the DNS name of your MongoDB instance. +This will be used by other MongoDB instances when forming a MongoDB +replica set. It should resolve to the MongoDB instance in your cluster when +you are done with the setup. This will help when we are adding more MongoDB +instances to the replica set in the future. + + +For ACS +^^^^^^^ +In Kubernetes on ACS, the name you populate in the ``data.fqdn`` field +will be used to configure a DNS name for the public IP assigned to the +Kubernetes Service that is the frontend for the MongoDB instance. + +We suggest using a name that will already be available in Azure. +We use ``mdb-instance-0``, ``mdb-instance-1`` and so on in this document, +which gives us ``mdb-instance-0..cloudapp.azure.com``, +``mdb-instance-1..cloudapp.azure.com``, etc. as the FQDNs. +The ```` is the Azure datacenter location you are using, +which can also be obtained using the ``az account list-locations`` command. + +You can also try to assign a name to an Public IP in Azure before starting +the process, or use ``nslookup`` with the name you have in mind to check +if it's available for use. + +In the rare chance that name in the ``data.fqdn`` field is not available, +we will need to create a ConfigMap with a unique name and restart the +MongoDB instance. + +For Kubernetes on bare-metal or other cloud providers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +On other environments, you need to provide the name resolution function +by other means (using DNS providers like GoDaddy, CloudFlare or your own +private DNS server). The DNS set up for other environments is currently +beyond the scope of this document. + + +Create the required ConfigMap using: + +.. code:: bash + + $ kubectl apply -f mongo-cm.yaml + + +You can check its status using: ``kubectl get cm`` + + + Now we are ready to run MongoDB and BigchainDB on our Kubernetes cluster. -Step 5: Run MongoDB as a StatefulSet +Step 6: Run MongoDB as a StatefulSet ------------------------------------ Get the file ``mongo-ss.yaml`` from GitHub using: @@ -188,7 +260,7 @@ To avoid this, we use the Docker feature of ``--cap-add=FOWNER``. This bypasses the uid and gid permission checks during writes and allows data to be persisted to disk. Refer to the -`Docker doc `_ +`Docker docs `_ for details. As we gain more experience running MongoDB in testing and production, we will @@ -205,8 +277,91 @@ Create the required StatefulSet using: You can check its status using the commands ``kubectl get statefulsets -w`` and ``kubectl get svc -w`` - -Step 6: Run BigchainDB as a Deployment +You may have to wait for upto 10 minutes wait for disk to be created +and attached on the first run. The pod can fail several times with the message +specifying that the timeout for mounting the disk has exceeded. + + +Step 7: Initialize a MongoDB Replica Set - Optional +--------------------------------------------------- + +This step is required only if you are planning to set up multiple +`BigchainDB nodes +`_, else you can +skip to the :ref:`step 9 `. + + +Login to the running MongoDB instance and access the mongo shell using: + +.. code:: bash + + $ kubectl exec -it mdb-0 -c mongodb -- /bin/bash + root@mdb-0:/# mongo --port 27017 + +We initialize the replica set by using the ``rs.initiate()`` command from the +mongo shell. Its syntax is: + +.. code:: bash + + rs.initiate({ + _id : ":" + } ] + }) + +An example command might look like: + +.. code:: bash + + > rs.initiate({ _id : "bigchain-rs", members: [ { _id : 0, host :"mdb-instance-0.westeurope.cloudapp.azure.com:27017" } ] }) + + +where ``mdb-instance-0.westeurope.cloudapp.azure.com`` is the value stored in +the ``data.fqdn`` field in the ConfigMap created using ``mongo-cm.yaml``. + + +You should see changes in the mongo shell prompt from ``>`` +to ``bigchain-rs:OTHER>`` to ``bigchain-rs:SECONDARY>`` and finally +to ``bigchain-rs:PRIMARY>``. + +You can use the ``rs.conf()`` and the ``rs.status()`` commands to check the +detailed replica set configuration now. + + +Step 8: Create a DNS record - Optional +-------------------------------------- + +This step is required only if you are planning to set up multiple +`BigchainDB nodes +`_, else you can +skip to the :ref:`next step `. + +Since we currently rely on Azure to provide us with a public IP and manage the +DNS entries of MongoDB instances, we detail only the steps required for ACS +here. + +Select the current Azure resource group and look for the ``Public IP`` +resource. You should see at least 2 entries there - one for the Kubernetes +master and the other for the MongoDB instance. You may have to ``Refresh`` the +Azure web page listing the resources in a resource group for the latest +changes to be reflected. + +Select the ``Public IP`` resource that is attached to your service (it should +have the Kubernetes cluster name alongwith a random string), +select ``Configuration``, add the DNS name that was added in the +ConfigMap earlier, click ``Save``, and wait for the changes to be applied. + +To verify the DNS setting is operational, you can run ``nslookup `` from your local Linux shell. + + +This will ensure that when you scale the replica set later, other MongoDB +members in the replica set can reach this instance. + + +Step 9: Run BigchainDB as a Deployment -------------------------------------- Get the file ``bigchaindb-dep.yaml`` from GitHub using: @@ -239,23 +394,23 @@ Create the required Deployment using: You can check its status using the command ``kubectl get deploy -w`` -Step 7: Verify the BigchainDB Node Setup ----------------------------------------- +Step 10: Verify the BigchainDB Node Setup +----------------------------------------- -Step 7.1: Testing Externally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Step 10.1: Testing Externally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Try to access the ``:9984`` on your -browser. You must receive a json output that shows the BigchainDB server -version among other things. +Try to access the ``:9984`` +on your browser. You must receive a json output that shows the BigchainDB +server version among other things. -Try to access the ``:27017`` on your -browser. You must receive a message from MongoDB stating that it doesn't allow -HTTP connections to the port anymore. +Try to access the ``:27017`` +on your browser. You must receive a message from MongoDB stating that it +doesn't allow HTTP connections to the port anymore. -Step 7.2: Testing Internally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Step 10.2: Testing Internally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Run a container that provides utilities like ``nslookup``, ``curl`` and ``dig`` on the cluster and query the internal DNS and IP endpoints. @@ -270,7 +425,7 @@ Now we can query for the ``mdb`` and ``bdb`` service details. .. code:: bash $ nslookup mdb - $ dig +noall +answer _mdb_port._tcp.mdb.default.svc.cluster.local SRV + $ dig +noall +answer _mdb-port._tcp.mdb.default.svc.cluster.local SRV $ curl -X GET http://mdb:27017 $ curl -X GET http://bdb:9984 diff --git a/k8s/deprecated.to.del/mongo-statefulset.yaml b/k8s/deprecated.to.del/mongo-statefulset.yaml new file mode 100644 index 00000000..a71567f3 --- /dev/null +++ b/k8s/deprecated.to.del/mongo-statefulset.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongodb + labels: + name: mongodb +spec: + ports: + - port: 27017 + targetPort: 27017 + clusterIP: None + selector: + role: mongodb +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: mongodb +spec: + serviceName: mongodb + replicas: 3 + template: + metadata: + labels: + role: mongodb + environment: staging + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongo + image: mongo:3.4.1 + command: + - mongod + - "--replSet" + - bigchain-rs + #- "--smallfiles" + #- "--noprealloc" + ports: + - containerPort: 27017 + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + - name: mongo-sidecar + image: cvallance/mongo-k8s-sidecar + env: + - name: MONGO_SIDECAR_POD_LABELS + value: "role=mongo,environment=staging" + volumeClaimTemplates: + - metadata: + name: mongo-persistent-storage + annotations: + volume.beta.kubernetes.io/storage-class: "fast" + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Gi diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile new file mode 100644 index 00000000..11fc80cf --- /dev/null +++ b/k8s/mongodb/container/Dockerfile @@ -0,0 +1,12 @@ +FROM mongo:3.4.2 +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY mongod.conf.template /etc/mongod.conf.template +COPY mongod_entrypoint/mongod_entrypoint / +VOLUME /data/db /data/configdb +EXPOSE 27017 +ENTRYPOINT ["/mongod_entrypoint"] diff --git a/k8s/mongodb/container/Makefile b/k8s/mongodb/container/Makefile new file mode 100644 index 00000000..72ec4f79 --- /dev/null +++ b/k8s/mongodb/container/Makefile @@ -0,0 +1,51 @@ +# Targets: +# all: Cleans, formats src files, builds the code, builds the docker image +# clean: Removes the binary and docker image +# format: Formats the src files +# build: Builds the code +# docker: Builds the code and docker image +# push: Push the docker image to Docker hub + +GOCMD=go +GOVET=$(GOCMD) tool vet +GOINSTALL=$(GOCMD) install +GOFMT=gofmt -s -w + +DOCKER_IMAGE_NAME?=bigchaindb/mongodb +DOCKER_IMAGE_TAG?=latest + +PWD=$(shell pwd) +BINARY_PATH=$(PWD)/mongod_entrypoint/ +BINARY_NAME=mongod_entrypoint +MAIN_FILE = $(BINARY_PATH)/mongod_entrypoint.go +SRC_FILES = $(BINARY_PATH)/mongod_entrypoint.go + +.PHONY: all + +all: clean build docker + +clean: + @echo "removing any pre-built binary"; + -@rm $(BINARY_PATH)/$(BINARY_NAME); + @echo "remove any pre-built docker image"; + -@docker rmi $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG); + +format: + $(GOFMT) $(SRC_FILES) + +build: format + $(shell cd $(BINARY_PATH) && \ + export GOPATH="$(BINARY_PATH)" && \ + export GOBIN="$(BINARY_PATH)" && \ + CGO_ENABLED=0 GOOS=linux $(GOINSTALL) -ldflags "-s" -a -installsuffix cgo $(MAIN_FILE)) + +docker: build + docker build \ + -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) .; + +vet: + $(GOVET) . + +push: + docker push \ + $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG); diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md new file mode 100644 index 00000000..7896a912 --- /dev/null +++ b/k8s/mongodb/container/README.md @@ -0,0 +1,88 @@ +## Custom MongoDB container for BigchainDB Backend + +### Need + +* MongoDB needs the hostname provided in the rs.initiate() command to be + resolvable through the hosts file locally. +* In the future, with the introduction of TLS for inter-cluster MongoDB + communications, we will need a way to specify detailed configuration. +* We also need a way to overwrite certain parameters to suit our use case. + + +### Step 1: Build the Latest Container + +`make` from the root of this project. + + +### Step 2: Run the Container + +``` +docker run \ +--name=mdb1 \ +--publish=17017:17017 \ +--rm=true \ +bigchaindb/mongodb \ +--replica-set-name \ +--fqdn \ +--port +``` + +#### Step 3: Initialize the Replica Set + +Login to one of the MongoDB containers, say mdb1: + +`docker exec -it mdb1 bash` + +Start the `mongo` shell: + +`mongo --port 27017` + + +Run the rs.initiate() command: +``` +rs.initiate({ + _id : ":" + } ] +}) +``` + +For example: + +``` +rs.initiate({ _id : "test-repl-set", members: [ { _id : 0, host : +"mdb-instance-0.westeurope.cloudapp.azure.com:27017" } ] }) +``` + +You should also see changes in the mongo shell prompt from `>` to +`test-repl-set:OTHER>` to `test-repl-set:SECONDARY>` to finally +`test-repl-set:PRIMARY>`. +If this instance is not the primary, you can use the `rs.status()` command to +find out who is the primary. + + +#### Step 4: Add members to the Replica Set + +We can only add members to a replica set from the PRIMARY instance. +Login to the PRIMARY and open a `mongo` shell. + +Run the rs.add() command with the ip and port number of the other +containers/instances: +``` +rs.add(":") +``` + +For example: + +Add mdb2 to replica set from mdb1: +``` +rs.add("bdb-cluster-1.northeurope.cloudapp.azure.com:27017") +``` + +Add mdb3 to replica set from mdb1: +``` +rs.add("bdb-cluster-2.northeurope.cloudapp.azure.com:27017") +``` + diff --git a/k8s/mongodb/container/mongod.conf.template b/k8s/mongodb/container/mongod.conf.template new file mode 100644 index 00000000..28e74acf --- /dev/null +++ b/k8s/mongodb/container/mongod.conf.template @@ -0,0 +1,89 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# where to write logging data. +systemLog: + verbosity: 0 + #TODO traceAllExceptions: true + timeStampFormat: iso8601-utc + component: + accessControl: + verbosity: 0 + command: + verbosity: 0 + control: + verbosity: 0 + ftdc: + verbosity: 0 + geo: + verbosity: 0 + index: + verbosity: 0 + network: + verbosity: 0 + query: + verbosity: 0 + replication: + verbosity: 0 + sharding: + verbosity: 0 + storage: + verbosity: 0 + journal: + verbosity: 0 + write: + verbosity: 0 + +processManagement: + fork: false + pidFilePath: /tmp/mongod.pid + +net: + port: PORT + bindIp: 0.0.0.0 + maxIncomingConnections: 8192 + wireObjectCheck: false + unixDomainSocket: + enabled: false + pathPrefix: /tmp + filePermissions: 0700 + http: + enabled: false + compression: + compressors: snappy + #ssl: TODO + +#security: TODO + +#setParameter: + #notablescan: 1 TODO + #logUserIds: 1 TODO + +storage: + dbPath: /data/db + indexBuildRetry: true + journal: + enabled: true + commitIntervalMs: 100 + directoryPerDB: true + engine: wiredTiger + wiredTiger: + engineConfig: + journalCompressor: snappy + collectionConfig: + blockCompressor: snappy + indexConfig: + prefixCompression: true # TODO false may affect performance? + +operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + +replication: + replSetName: REPLICA_SET_NAME + enableMajorityReadConcern: true + +#sharding: + diff --git a/k8s/mongodb/container/mongod_entrypoint/mongod_entrypoint.go b/k8s/mongodb/container/mongod_entrypoint/mongod_entrypoint.go new file mode 100644 index 00000000..57b48974 --- /dev/null +++ b/k8s/mongodb/container/mongod_entrypoint/mongod_entrypoint.go @@ -0,0 +1,154 @@ +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "regexp" + "syscall" +) + +const ( + mongoConfFilePath string = "/etc/mongod.conf" + mongoConfTemplateFilePath string = "/etc/mongod.conf.template" + hostsFilePath string = "/etc/hosts" +) + +var ( + // Use the same entrypoint as the mongo:3.4.2 image; just supply it with + // the mongod conf file with custom params + mongoStartCmd []string = []string{"/entrypoint.sh", "mongod", "--config", + mongoConfFilePath} +) + +// context struct stores the user input and the constraints for the specified +// input. It also stores the keyword that needs to be replaced in the template +// files. +type context struct { + cliInput string + templateKeyword string + regex string +} + +// sanity function takes the pre-defined constraints and the user inputs as +// arguments and validates user input based on regex matching +func sanity(input map[string]*context, fqdn, ip string) error { + var format *regexp.Regexp + for _, ctx := range input { + format = regexp.MustCompile(ctx.regex) + if format.MatchString(ctx.cliInput) == false { + return errors.New(fmt.Sprintf( + "Invalid value: '%s' for '%s'. Can be '%s'", + ctx.cliInput, + ctx.templateKeyword, + ctx.regex)) + } + } + + format = regexp.MustCompile(`[a-z0-9-.]+`) + if format.MatchString(fqdn) == false { + return errors.New(fmt.Sprintf( + "Invalid value: '%s' for FQDN. Can be '%s'", + fqdn, + format)) + } + + if net.ParseIP(ip) == nil { + return errors.New(fmt.Sprintf( + "Invalid value: '%s' for IPv4. Can be a.b.c.d", + ip)) + } + + return nil +} + +// createFile function takes the pre-defined keywords, user inputs, the +// template file path and the new file path location as parameters, and +// creates a new file at file path with all the keywords replaced by inputs. +func createFile(input map[string]*context, + template string, conf string) error { + // read the template + contents, err := ioutil.ReadFile(template) + if err != nil { + return err + } + // replace + for _, ctx := range input { + contents = bytes.Replace(contents, []byte(ctx.templateKeyword), + []byte(ctx.cliInput), -1) + } + // write + err = ioutil.WriteFile(conf, contents, 0644) + if err != nil { + return err + } + return nil +} + +// updateHostsFile takes the FQDN supplied as input to the container and adds +// an entry to /etc/hosts +func updateHostsFile(ip, fqdn string) error { + fileHandle, err := os.OpenFile(hostsFilePath, os.O_APPEND|os.O_WRONLY, + os.ModeAppend) + if err != nil { + return err + } + defer fileHandle.Close() + // append + _, err = fileHandle.WriteString(fmt.Sprintf("\n%s %s\n", ip, fqdn)) + if err != nil { + return err + } + return nil +} + +func main() { + var fqdn, ip string + input := make(map[string]*context) + + input["replica-set-name"] = &context{} + input["replica-set-name"].regex = `[a-z]+` + input["replica-set-name"].templateKeyword = "REPLICA_SET_NAME" + flag.StringVar(&input["replica-set-name"].cliInput, + "replica-set-name", + "", + "replica set name") + + input["port"] = &context{} + input["port"].regex = `[0-9]{4,5}` + input["port"].templateKeyword = "PORT" + flag.StringVar(&input["port"].cliInput, + "port", + "", + "mongodb port number") + + flag.StringVar(&fqdn, "fqdn", "", "FQDN of the MongoDB instance") + flag.StringVar(&ip, "ip", "", "IPv4 address of the container") + + flag.Parse() + err := sanity(input, fqdn, ip) + if err != nil { + log.Fatal(err) + } + + err = createFile(input, mongoConfTemplateFilePath, mongoConfFilePath) + if err != nil { + log.Fatal(err) + } + + err = updateHostsFile(ip, fqdn) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Starting Mongod....") + err = syscall.Exec(mongoStartCmd[0], mongoStartCmd[0:], os.Environ()) + if err != nil { + panic(err) + } +} diff --git a/k8s/mongodb/mongo-cm.yaml b/k8s/mongodb/mongo-cm.yaml new file mode 100644 index 00000000..bf4b4f82 --- /dev/null +++ b/k8s/mongodb/mongo-cm.yaml @@ -0,0 +1,13 @@ +##################################################################### +# This YAML file desribes a ConfigMap with the FQDN of the mongo # +# instance to be started. MongoDB instance uses the value from this # +# ConfigMap to bootstrap itself during startup. # +##################################################################### + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mdb-fqdn + namespace: default +data: + fqdn: mdb-instance-0.westeurope.cloudapp.azure.com diff --git a/k8s/mongodb/mongo-data-configdb-pvc.yaml b/k8s/mongodb/mongo-data-configdb-pvc.yaml deleted file mode 100644 index 7d3dc8a3..00000000 --- a/k8s/mongodb/mongo-data-configdb-pvc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -########################################################## -# This YAML file desribes a k8s pvc for mongodb configDB # -########################################################## - -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: mongo-configdb-claim - annotations: - volume.beta.kubernetes.io/storage-class: slow-configdb -spec: - accessModes: - - ReadWriteOnce - # FIXME(Uncomment when ACS supports this!) - # persistentVolumeReclaimPolicy: Retain - resources: - requests: - storage: 20Gi diff --git a/k8s/mongodb/mongo-data-configdb-sc.yaml b/k8s/mongodb/mongo-data-configdb-sc.yaml deleted file mode 100644 index b431db67..00000000 --- a/k8s/mongodb/mongo-data-configdb-sc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -################################################################### -# This YAML file desribes a StorageClass for the mongodb configDB # -################################################################### - -kind: StorageClass -apiVersion: storage.k8s.io/v1beta1 -metadata: - name: slow-configdb -provisioner: kubernetes.io/azure-disk -parameters: - skuName: Standard_LRS - location: westeurope diff --git a/k8s/mongodb/mongo-data-db-pvc.yaml b/k8s/mongodb/mongo-data-db-pvc.yaml deleted file mode 100644 index e9689346..00000000 --- a/k8s/mongodb/mongo-data-db-pvc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -######################################################## -# This YAML file desribes a k8s pvc for mongodb dbPath # -######################################################## - -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: mongo-db-claim - annotations: - volume.beta.kubernetes.io/storage-class: slow-db -spec: - accessModes: - - ReadWriteOnce - # FIXME(Uncomment when ACS supports this!) - # persistentVolumeReclaimPolicy: Retain - resources: - requests: - storage: 20Gi diff --git a/k8s/mongodb/mongo-data-db-sc.yaml b/k8s/mongodb/mongo-data-db-sc.yaml deleted file mode 100644 index f700223d..00000000 --- a/k8s/mongodb/mongo-data-db-sc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -################################################################# -# This YAML file desribes a StorageClass for the mongodb dbPath # -################################################################# - -kind: StorageClass -apiVersion: storage.k8s.io/v1beta1 -metadata: - name: slow-db -provisioner: kubernetes.io/azure-disk -parameters: - skuName: Standard_LRS - location: westeurope diff --git a/k8s/mongodb/mongo-pvc.yaml b/k8s/mongodb/mongo-pvc.yaml new file mode 100644 index 00000000..da257527 --- /dev/null +++ b/k8s/mongodb/mongo-pvc.yaml @@ -0,0 +1,35 @@ +########################################################### +# This section file desribes a k8s pvc for mongodb dbPath # +########################################################### +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mongo-db-claim + annotations: + volume.beta.kubernetes.io/storage-class: slow-db +spec: + accessModes: + - ReadWriteOnce + # FIXME(Uncomment when ACS supports this!) + # persistentVolumeReclaimPolicy: Retain + resources: + requests: + storage: 20Gi +--- +############################################################# +# This YAML section desribes a k8s pvc for mongodb configDB # +############################################################# +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mongo-configdb-claim + annotations: + volume.beta.kubernetes.io/storage-class: slow-configdb +spec: + accessModes: + - ReadWriteOnce + # FIXME(Uncomment when ACS supports this!) + # persistentVolumeReclaimPolicy: Retain + resources: + requests: + storage: 1Gi diff --git a/k8s/mongodb/mongo-sc.yaml b/k8s/mongodb/mongo-sc.yaml new file mode 100644 index 00000000..2f291ffe --- /dev/null +++ b/k8s/mongodb/mongo-sc.yaml @@ -0,0 +1,23 @@ +#################################################################### +# This YAML section desribes a StorageClass for the mongodb dbPath # +#################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow-db +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Standard_LRS + location: westeurope +--- +###################################################################### +# This YAML section desribes a StorageClass for the mongodb configDB # +###################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow-configdb +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Standard_LRS + location: westeurope diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index 63c7d27d..fb6a73f8 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -37,14 +37,30 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: mongodb - image: mongo:3.4.1 + # TODO(FIXME): Do not use latest in production as it is harder to track + # versions during updates and rollbacks. Also, once fixed, change the + # imagePullPolicy to IfNotPresent for faster bootup + image: bigchaindb/mongodb:latest + env: + - name: MONGODB_FQDN + valueFrom: + configMapKeyRef: + name: mdb-fqdn + key: fqdn + - name: MONGODB_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP args: - - --replSet=bigchain-rs + - --replica-set-name=bigchain-rs + - --fqdn=$(MONGODB_FQDN) + - --port=27017 + - --ip=$(MONGODB_POD_IP) securityContext: capabilities: add: - FOWNER - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - containerPort: 27017 hostPort: 27017 From f00f68e03fe73950ceabe606ac81a527188ae2bf Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 16 Mar 2017 14:10:04 +0100 Subject: [PATCH 06/80] _sign_threshold now signs all subconditions for a public key. Created test. --- bigchaindb/common/transaction.py | 26 +++++++++++------------ tests/common/test_transaction.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 23b8f169..e956812f 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -768,20 +768,19 @@ class Transaction(object): key_pairs (dict): The keys to sign the Transaction with. """ input_ = deepcopy(input_) - for owner_before in input_.owners_before: - try: - # TODO: CC should throw a KeypairMismatchException, instead of - # our manual mapping here + for owner_before in set(input_.owners_before): + # TODO: CC should throw a KeypairMismatchException, instead of + # our manual mapping here - # TODO FOR CC: Naming wise this is not so smart, - # `get_subcondition` in fact doesn't return a - # condition but a fulfillment + # TODO FOR CC: Naming wise this is not so smart, + # `get_subcondition` in fact doesn't return a + # condition but a fulfillment - # TODO FOR CC: `get_subcondition` is singular. One would not - # expect to get a list back. - ccffill = input_.fulfillment - subffill = ccffill.get_subcondition_from_vk(owner_before)[0] - except IndexError: + # TODO FOR CC: `get_subcondition` is singular. One would not + # expect to get a list back. + ccffill = input_.fulfillment + subffills = ccffill.get_subcondition_from_vk(owner_before) + if not subffills: raise KeypairMismatchException('Public key {} cannot be found ' 'in the fulfillment' .format(owner_before)) @@ -794,7 +793,8 @@ class Transaction(object): # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings - subffill.sign(tx_serialized.encode(), private_key) + for subffill in subffills: + subffill.sign(tx_serialized.encode(), private_key) self.inputs[index] = input_ def inputs_valid(self, outputs=None): diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 45cadc3b..f74e535e 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -590,6 +590,42 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_input, validate_transaction_model(tx) +def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, + asset_definition): + from copy import deepcopy + from cryptoconditions import Ed25519Fulfillment, ThresholdSha256Fulfillment + from bigchaindb.common.transaction import Input, Output, Transaction + from bigchaindb.common.crypto import PrivateKey + + threshold = ThresholdSha256Fulfillment(threshold=2) + threshold.add_subfulfillment(Ed25519Fulfillment(public_key=user_pub)) + threshold.add_subfulfillment(Ed25519Fulfillment(public_key=user_pub)) + + threshold_input = Input(threshold, [user_pub, user_pub]) + threshold_output = Output(threshold, [user_pub, user_pub]) + + tx = Transaction(Transaction.CREATE, asset_definition, + [threshold_input], [threshold_output]) + expected = deepcopy(threshold_input) + expected.fulfillment.subconditions[0]['body'].sign(str(tx).encode(), + PrivateKey(user_priv)) + expected.fulfillment.subconditions[1]['body'].sign(str(tx).encode(), + PrivateKey(user_priv)) + + tx.sign([user_priv, user_priv]) + + subconditions = tx.inputs[0].fulfillment.subconditions + expected_subconditions = expected.fulfillment.subconditions + assert subconditions[0]['body'].to_dict()['signature'] == \ + expected_subconditions[0]['body'].to_dict()['signature'] + assert subconditions[1]['body'].to_dict()['signature'] == \ + expected_subconditions[1]['body'].to_dict()['signature'] + + assert tx.inputs[0].to_dict()['fulfillment'] == \ + expected.fulfillment.serialize_uri() + assert tx.inputs_valid() is True + + def test_multiple_input_validation_of_transfer_tx(user_input, user_output, user_priv, user2_pub, user2_priv, user3_pub, From 4aa6ed106710239d24b6f1fc1bc93bd5837cf7d9 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 16 Mar 2017 14:18:57 +0100 Subject: [PATCH 07/80] fixed pep8 error --- tests/common/test_transaction.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index f74e535e..16ba34e6 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -617,12 +617,12 @@ def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, subconditions = tx.inputs[0].fulfillment.subconditions expected_subconditions = expected.fulfillment.subconditions assert subconditions[0]['body'].to_dict()['signature'] == \ - expected_subconditions[0]['body'].to_dict()['signature'] + expected_subconditions[0]['body'].to_dict()['signature'] assert subconditions[1]['body'].to_dict()['signature'] == \ - expected_subconditions[1]['body'].to_dict()['signature'] + expected_subconditions[1]['body'].to_dict()['signature'] assert tx.inputs[0].to_dict()['fulfillment'] == \ - expected.fulfillment.serialize_uri() + expected.fulfillment.serialize_uri() assert tx.inputs_valid() is True From 7aa94447cd9e018b5cbf26263adfd8c1ba7c8699 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 16 Mar 2017 14:42:32 +0100 Subject: [PATCH 08/80] docs: copyedited 2 pages re/ node on k8s --- .../add-node-on-kubernetes.rst | 79 +++++++-------- .../node-on-kubernetes.rst | 99 +++++++------------ 2 files changed, 73 insertions(+), 105 deletions(-) diff --git a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst index 542d3d2b..ea435ed3 100644 --- a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst @@ -1,25 +1,26 @@ -Add a BigchainDB Node in a Kubernetes Cluster -============================================= +Kubernetes Template: Add a BigchainDB Node to an Existing BigchainDB Cluster +============================================================================ -**Refer this document if you want to add a new BigchainDB node to an existing -cluster** +This page describes how to deploy a BigchainDB node using Kubernetes, +and how to add that node to an existing BigchainDB cluster. +It assumes you already have a running Kubernetes cluster +where you can deploy the new BigchainDB node. -**If you want to start your first BigchainDB node in the BigchainDB cluster, -refer** -:doc:`this ` +If you want to deploy the first BigchainDB node in a BigchainDB cluster, +or a stand-alone BigchainDB node, +then see :doc:`the page about that `. Terminology Used ---------------- -``existing cluster`` will refer to the existing (or any one of the existing) -Kubernetes cluster that already hosts a BigchainDB instance with a MongoDB -backend. +``existing cluster`` will refer to one of the existing Kubernetes clusters +hosting one of the existing BigchainDB nodes. ``ctx-1`` will refer to the kubectl context of the existing cluster. ``new cluster`` will refer to the new Kubernetes cluster that will run a new -BigchainDB instance with a MongoDB backend. +BigchainDB node (including a BigchainDB instance and a MongoDB instance). ``ctx-2`` will refer to the kubectl context of the new cluster. @@ -38,26 +39,19 @@ existing cluster. Step 1: Prerequisites --------------------- -* You will need to have a public and private key for the new BigchainDB - instance you will set up. +* A public/private key pair for the new BigchainDB instance. * The public key should be shared offline with the other existing BigchainDB - instances. The means to achieve this requirement is beyond the scope of this - document. + nodes in the existing BigchainDB cluster. -* You will need the public keys of all the existing BigchainDB instances. The - means to achieve this requirement is beyond the scope of this document. +* You will need the public keys of all the existing BigchainDB nodes. * A new Kubernetes cluster setup with kubectl configured to access it. - If you are using Kubernetes on Azure Container Server (ACS), please refer - our documentation `here ` for the set up. -If you haven't read our guide to set up a -:doc:`node on Kubernetes `, now is a good time to jump in -there and then come back here as these instructions build up from there. +* Some familiarity with deploying a BigchainDB node on Kubernetes. + See our :doc:`other docs about that `. - -NOTE: If you are managing multiple kubernetes clusters, from your local +Note: If you are managing multiple Kubernetes clusters, from your local system, you can run ``kubectl config view`` to list all the contexts that are available for the local kubectl. To target a specific cluster, add a ``--context`` flag to the kubectl CLI. For @@ -71,9 +65,10 @@ example: $ kubectl --context ctx-2 proxy --port 8002 -Step 2: Prepare the New Kubernetes cluster +Step 2: Prepare the New Kubernetes Cluster ------------------------------------------ -Follow the steps in the sections to set up Storage Classes and Persisten Volume + +Follow the steps in the sections to set up Storage Classes and Persistent Volume Claims, and to run MongoDB in the new cluster: 1. :ref:`Add Storage Classes ` @@ -84,13 +79,13 @@ Claims, and to run MongoDB in the new cluster: Step 3: Add the New MongoDB Instance to the Existing Replica Set ---------------------------------------------------------------- -Note that by ``replica set`` we are referring to the MongoDB replica set, and not -to Kubernetes' ``ReplicaSet``. -If you are not the administrator of an existing MongoDB/BigchainDB instance, you -will have to coordinate offline with an existing administrator so that s/he can -add the new MongoDB instance to the replica set. The means to achieve this is -beyond the scope of this document. +Note that by ``replica set``, we are referring to the MongoDB replica set, +not a Kubernetes' ``ReplicaSet``. + +If you are not the administrator of an existing BigchainDB node, you +will have to coordinate offline with an existing administrator so that they can +add the new MongoDB instance to the replica set. Add the new instance of MongoDB from an existing instance by accessing the ``mongo`` shell. @@ -100,7 +95,7 @@ Add the new instance of MongoDB from an existing instance by accessing the $ kubectl --context ctx-1 exec -it mdb-0 -c mongodb -- /bin/bash root@mdb-0# mongo --port 27017 -We can only add members to a replica set from the ``PRIMARY`` instance. +One can only add members to a replica set from the ``PRIMARY`` instance. The ``mongo`` shell prompt should state that this is the primary member in the replica set. If not, then you can use the ``rs.status()`` command to find out who the @@ -113,7 +108,7 @@ Run the ``rs.add()`` command with the FQDN and port number of the other instance PRIMARY> rs.add(":") -Step 4: Verify the replica set membership +Step 4: Verify the Replica Set Membership ----------------------------------------- You can use the ``rs.conf()`` and the ``rs.status()`` commands available in the @@ -123,7 +118,7 @@ The new MongoDB instance should be listed in the membership information displayed. -Step 5: Start the new BigchainDB instance +Step 5: Start the New BigchainDB Instance ----------------------------------------- Get the file ``bigchaindb-dep.yaml`` from GitHub using: @@ -149,20 +144,20 @@ Create the required Deployment using: You can check its status using the command ``kubectl get deploy -w`` -Step 6: Restart the existing BigchainDB instance(s) +Step 6: Restart the Existing BigchainDB Instance(s) --------------------------------------------------- -Add public key of the new BigchainDB instance to the keyring of all the -existing instances and update the BigchainDB instances using: + +Add the public key of the new BigchainDB instance to the keyring of all the +existing BigchainDB instances and update the BigchainDB instances using: .. code:: bash $ kubectl --context ctx-1 replace -f bigchaindb-dep.yaml -This will create a ``rolling deployment`` in Kubernetes where a new instance of +This will create a "rolling deployment" in Kubernetes where a new instance of BigchainDB will be created, and if the health check on the new instance is successful, the earlier one will be terminated. This ensures that there is zero downtime during updates. -You can login to an existing BigchainDB instance and run the ``bigchaindb -show-config`` command to see the configuration update to the keyring. - +You can SSH to an existing BigchainDB instance and run the ``bigchaindb +show-config`` command to check that the keyring is updated. diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst index 650d2f45..b19d79a3 100644 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -1,15 +1,13 @@ -Bootstrap a BigchainDB Node in a Kubernetes Cluster -=================================================== +Kubernetes Template: Deploy a Single BigchainDB Node +==================================================== -**Refer this document if you are starting your first BigchainDB instance in -a BigchainDB cluster or starting a stand-alone BigchainDB instance** +This page describes how to deploy the first BigchainDB node +in a BigchainDB cluster, or a stand-alone BigchainDB node, +using `Kubernetes `_. +It assumes you already have a running Kubernetes cluster. -**If you want to add a new BigchainDB node to an existing cluster, refer** -:doc:`this ` - -Assuming you already have a `Kubernetes `_ -cluster up and running, this page describes how to run a -BigchainDB node in it. +If you want to add a new BigchainDB node to an existing BigchainDB cluster, +refer to :doc:`the page about that `. Step 1: Install kubectl @@ -49,18 +47,17 @@ Step 3: Create Storage Classes MongoDB needs somewhere to store its data persistently, outside the container where MongoDB is running. - -The official MongoDB Docker container exports two volume mounts with correct +Our MongoDB Docker container +(based on the official MongoDB Docker container) +exports two volume mounts with correct permissions from inside the container: +* The directory where the mongod instance stores its data: ``/data/db``. + There's more explanation in the MongoDB docs about `storage.dbpath `_. -* The directory where the mongod instance stores its data - ``/data/db``, - described at `storage.dbpath `_. - -* The directory where mongodb instance stores the metadata for a sharded - cluster - ``/data/configdb/``, described at - `sharding.configDB `_. - +* The directory where the mongodb instance stores the metadata for a sharded + cluster: ``/data/configdb/``. + There's more explanation in the MongoDB docs about `sharding.configDB `_. Explaining how Kubernetes handles persistent volumes, and the associated terminology, @@ -69,9 +66,6 @@ see `the Kubernetes docs about persistent volumes `_. The first thing to do is create the Kubernetes storage classes. -We will accordingly create two storage classes and persistent volume claims in -Kubernetes. - **Azure.** First, you need an Azure storage account. If you deployed your Kubernetes cluster on Azure @@ -85,7 +79,6 @@ Standard storage is lower-cost and lower-performance. It uses hard disk drives (HDD). LRS means locally-redundant storage: three replicas in the same data center. - Premium storage is higher-cost and higher-performance. It uses solid state drives (SSD). At the time of writing, @@ -102,11 +95,10 @@ Get the file ``mongo-sc.yaml`` from GitHub using: $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-sc.yaml -You may want to update the ``parameters.location`` field in both the files to +You may have to update the ``parameters.location`` field in both the files to specify the location you are using in Azure. - -Create the required storage classes using +Create the required storage classes using: .. code:: bash @@ -115,7 +107,7 @@ Create the required storage classes using You can check if it worked using ``kubectl get storageclasses``. -Note that there is no line of the form +**Azure.** Note that there is no line of the form ``storageAccount: `` under ``parameters:``. When we included one and then created a PersistentVolumeClaim based on it, @@ -128,9 +120,8 @@ with the specified skuName and location. Step 4: Create Persistent Volume Claims --------------------------------------- -Next, we'll create two PersistentVolumeClaim objects ``mongo-db-claim`` and +Next, you will create two PersistentVolumeClaim objects ``mongo-db-claim`` and ``mongo-configdb-claim``. - Get the file ``mongo-pvc.yaml`` from GitHub using: .. code:: bash @@ -166,15 +157,14 @@ Step 5: Create the Config Map - Optional This step is required only if you are planning to set up multiple `BigchainDB nodes -`_, else you can -skip to the :ref:`next step `. +`_. MongoDB reads the local ``/etc/hosts`` file while bootstrapping a replica set to resolve the hostname provided to the ``rs.initiate()`` command. It needs to ensure that the replica set is being initialized in the same instance where the MongoDB instance is running. -To achieve this, we create a ConfigMap with the FQDN of the MongoDB instance +To achieve this, you will create a ConfigMap with the FQDN of the MongoDB instance and populate the ``/etc/hosts`` file with this value so that a replica set can be created seamlessly. @@ -188,35 +178,29 @@ You may want to update the ``data.fqdn`` field in the file before creating the ConfigMap. ``data.fqdn`` field will be the DNS name of your MongoDB instance. This will be used by other MongoDB instances when forming a MongoDB replica set. It should resolve to the MongoDB instance in your cluster when -you are done with the setup. This will help when we are adding more MongoDB +you are done with the setup. This will help when you are adding more MongoDB instances to the replica set in the future. -For ACS -^^^^^^^ +**Azure.** In Kubernetes on ACS, the name you populate in the ``data.fqdn`` field will be used to configure a DNS name for the public IP assigned to the Kubernetes Service that is the frontend for the MongoDB instance. - We suggest using a name that will already be available in Azure. We use ``mdb-instance-0``, ``mdb-instance-1`` and so on in this document, which gives us ``mdb-instance-0..cloudapp.azure.com``, ``mdb-instance-1..cloudapp.azure.com``, etc. as the FQDNs. The ```` is the Azure datacenter location you are using, which can also be obtained using the ``az account list-locations`` command. - You can also try to assign a name to an Public IP in Azure before starting the process, or use ``nslookup`` with the name you have in mind to check if it's available for use. - In the rare chance that name in the ``data.fqdn`` field is not available, -we will need to create a ConfigMap with a unique name and restart the +you must create a ConfigMap with a unique name and restart the MongoDB instance. -For Kubernetes on bare-metal or other cloud providers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On other environments, you need to provide the name resolution function +**Kubernetes on bare-metal or other cloud providers.** +You need to provide the name resolution function by other means (using DNS providers like GoDaddy, CloudFlare or your own private DNS server). The DNS set up for other environments is currently beyond the scope of this document. @@ -231,10 +215,9 @@ Create the required ConfigMap using: You can check its status using: ``kubectl get cm`` +Now you are ready to run MongoDB and BigchainDB on our Kubernetes cluster. -Now we are ready to run MongoDB and BigchainDB on our Kubernetes cluster. - Step 6: Run MongoDB as a StatefulSet ------------------------------------ @@ -250,12 +233,10 @@ Note how the MongoDB container uses the ``mongo-db-claim`` and the ``/data/configdb`` diretories (mount path). Note also that we use the pod's ``securityContext.capabilities.add`` specification to add the ``FOWNER`` capability to the container. - That is because MongoDB container has the user ``mongodb``, with uid ``999`` and group ``mongodb``, with gid ``999``. When this container runs on a host with a mounted disk, the writes fail when there is no user with uid ``999``. - To avoid this, we use the Docker feature of ``--cap-add=FOWNER``. This bypasses the uid and gid permission checks during writes and allows data to be persisted to disk. @@ -277,9 +258,9 @@ Create the required StatefulSet using: You can check its status using the commands ``kubectl get statefulsets -w`` and ``kubectl get svc -w`` -You may have to wait for upto 10 minutes wait for disk to be created +You may have to wait for up to 10 minutes for the disk to be created and attached on the first run. The pod can fail several times with the message -specifying that the timeout for mounting the disk has exceeded. +saying that the timeout for mounting the disk was exceeded. Step 7: Initialize a MongoDB Replica Set - Optional @@ -287,8 +268,7 @@ Step 7: Initialize a MongoDB Replica Set - Optional This step is required only if you are planning to set up multiple `BigchainDB nodes -`_, else you can -skip to the :ref:`step 9 `. +`_. Login to the running MongoDB instance and access the mongo shell using: @@ -298,7 +278,7 @@ Login to the running MongoDB instance and access the mongo shell using: $ kubectl exec -it mdb-0 -c mongodb -- /bin/bash root@mdb-0:/# mongo --port 27017 -We initialize the replica set by using the ``rs.initiate()`` command from the +You will initiate the replica set by using the ``rs.initiate()`` command from the mongo shell. Its syntax is: .. code:: bash @@ -335,28 +315,21 @@ Step 8: Create a DNS record - Optional This step is required only if you are planning to set up multiple `BigchainDB nodes -`_, else you can -skip to the :ref:`next step `. +`_. -Since we currently rely on Azure to provide us with a public IP and manage the -DNS entries of MongoDB instances, we detail only the steps required for ACS -here. - -Select the current Azure resource group and look for the ``Public IP`` +**Azure.** Select the current Azure resource group and look for the ``Public IP`` resource. You should see at least 2 entries there - one for the Kubernetes master and the other for the MongoDB instance. You may have to ``Refresh`` the Azure web page listing the resources in a resource group for the latest changes to be reflected. - Select the ``Public IP`` resource that is attached to your service (it should -have the Kubernetes cluster name alongwith a random string), +have the Kubernetes cluster name along with a random string), select ``Configuration``, add the DNS name that was added in the ConfigMap earlier, click ``Save``, and wait for the changes to be applied. To verify the DNS setting is operational, you can run ``nslookup `` from your local Linux shell. - This will ensure that when you scale the replica set later, other MongoDB members in the replica set can reach this instance. @@ -420,7 +393,7 @@ on the cluster and query the internal DNS and IP endpoints. $ kubectl run -it toolbox -- image --restart=Never --rm It will drop you to the shell prompt. -Now we can query for the ``mdb`` and ``bdb`` service details. +Now you can query for the ``mdb`` and ``bdb`` service details. .. code:: bash From b849656d3bbbc08fdb6e7e5fd06cb5a5636dd51a Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 14 Mar 2017 11:47:44 +0100 Subject: [PATCH 09/80] Use namedtuple for key pair --- bigchaindb/common/crypto.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/crypto.py b/bigchaindb/common/crypto.py index acce02d9..99663fe9 100644 --- a/bigchaindb/common/crypto.py +++ b/bigchaindb/common/crypto.py @@ -1,18 +1,31 @@ # Separate all crypto code so that we can easily test several implementations +from collections import namedtuple import sha3 from cryptoconditions import crypto +CryptoKeypair = namedtuple('CryptoKeypair', ('private_key', 'public_key')) + + def hash_data(data): """Hash the provided data using SHA3-256""" return sha3.sha3_256(data.encode()).hexdigest() def generate_key_pair(): + """Generates a cryptographic key pair. + + Returns: + :class:`~bigchaindb.common.crypto.CryptoKeypair`: A + :obj:`collections.namedtuple` with named fields + :attr:`~bigchaindb.common.crypto.CryptoKeypair.private_key` and + :attr:`~bigchaindb.common.crypto.CryptoKeypair.public_key`. + + """ # TODO FOR CC: Adjust interface so that this function becomes unnecessary - private_key, public_key = crypto.ed25519_generate_key_pair() - return private_key.decode(), public_key.decode() + return CryptoKeypair( + *(k.decode() for k in crypto.ed25519_generate_key_pair())) PrivateKey = crypto.Ed25519SigningKey From 102406dd357ba8a14c85d32c99ea16d05bd9b521 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 16 Mar 2017 14:05:09 +0100 Subject: [PATCH 10/80] Add fixtures for alice, bob, and carol --- tests/conftest.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 9612f38b..e943d0a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -223,6 +223,54 @@ def user2_pk(): return USER2_PK +@pytest.fixture +def alice(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def alice_privkey(alice): + return alice.private_key + + +@pytest.fixture +def alice_pubkey(alice): + return alice.public_key + + +@pytest.fixture +def bob(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def bob_privkey(bob): + return bob.private_key + + +@pytest.fixture +def bob_pubkey(carol): + return bob.public_key + + +@pytest.fixture +def carol(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def carol_privkey(carol): + return carol.private_key + + +@pytest.fixture +def carol_pubkey(carol): + return carol.public_key + + @pytest.fixture def b(): from bigchaindb import Bigchain From c12f08a92c3c16cc2078d95eedbf5fcada923095 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 16 Mar 2017 14:01:43 +0100 Subject: [PATCH 11/80] Add test to reproduce false double spend see issue #1271 --- tests/test_core.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index f939ad05..b8803e9b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -80,3 +80,46 @@ def test_get_blocks_status_containing_tx(monkeypatch): bigchain = Bigchain(public_key='pubkey', private_key='privkey') with pytest.raises(Exception): bigchain.get_blocks_status_containing_tx('txid') + + +@pytest.mark.genesis +def test_get_spent_issue_1271(b, alice, bob, carol): + from bigchaindb.models import Transaction + + tx_1 = Transaction.create( + [carol.public_key], + [([carol.public_key], 8)], + ).sign([carol.private_key]) + + tx_2 = Transaction.transfer( + tx_1.to_inputs(), + [([bob.public_key], 2), + ([alice.public_key], 2), + ([carol.public_key], 4)], + asset_id=tx_1.id, + ).sign([carol.private_key]) + + tx_3 = Transaction.transfer( + tx_2.to_inputs()[2:3], + [([alice.public_key], 1), + ([carol.public_key], 3)], + asset_id=tx_1.id, + ).sign([carol.private_key]) + + tx_4 = Transaction.transfer( + tx_2.to_inputs()[1:2] + tx_3.to_inputs()[0:1], + [([bob.public_key], 3)], + asset_id=tx_1.id, + ).sign([alice.private_key]) + + tx_5 = Transaction.transfer( + tx_2.to_inputs()[0:1], + [([alice.public_key], 2)], + asset_id=tx_1.id, + ).sign([bob.private_key]) + block_5 = b.create_block([tx_1, tx_2, tx_3, tx_4, tx_5]) + b.write_block(block_5) + assert b.get_spent(tx_2.id, 0) == tx_5 + assert not b.get_spent(tx_5.id, 0) + assert b.get_outputs_filtered(alice.public_key) + assert b.get_outputs_filtered(alice.public_key, include_spent=False) From c5bad99f4ea8b5bbe9f8aa2085d7af38bcd6128d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 16 Mar 2017 13:40:25 +0100 Subject: [PATCH 12/80] Add test for get_spent for tx with two inputs --- tests/backend/mongodb/test_queries.py | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index 80e3cc91..1d7bfc39 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -159,6 +159,43 @@ def test_get_spent(signed_create_tx, signed_transfer_tx): assert spents[0] == signed_transfer_tx.to_dict() +def test_get_spent_for_tx_with_multiple_inputs(carol): + from bigchaindb.backend import connect, query + from bigchaindb.models import Block, Transaction + conn = connect() + tx_0 = Transaction.create( + [carol.public_key], + [([carol.public_key], 1), + ([carol.public_key], 1), + ([carol.public_key], 2)], + ).sign([carol.private_key]) + block = Block(transactions=[tx_0]) + conn.db.bigchain.insert_one(block.to_dict()) + spents = list(query.get_spent(conn, tx_0.id, 0)) + assert not spents + + tx_1 = Transaction.transfer( + tx_0.to_inputs()[2:3], + [([carol.public_key], 1), + ([carol.public_key], 1)], + asset_id=tx_0.id, + ).sign([carol.private_key]) + block = Block(transactions=[tx_1]) + conn.db.bigchain.insert_one(block.to_dict()) + spents = list(query.get_spent(conn, tx_0.id, 0)) + assert not spents + + tx_2 = Transaction.transfer( + tx_0.to_inputs()[0:1] + tx_1.to_inputs()[1:2], + [([carol.public_key], 2)], + asset_id=tx_0.id, + ).sign([carol.private_key]) + block = Block(transactions=[tx_2]) + conn.db.bigchain.insert_one(block.to_dict()) + spents = list(query.get_spent(conn, tx_0.id, 1)) + assert not spents + + def test_get_owned_ids(signed_create_tx, user_pk): from bigchaindb.backend import connect, query from bigchaindb.models import Block From b94a9ec7e9cf4d9009920e10b1420891d3e3f89d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 16 Mar 2017 13:39:34 +0100 Subject: [PATCH 13/80] Fix get_spent mongodb-based query fixes #1271 --- bigchaindb/backend/mongodb/query.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 1988db04..74b9c35a 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -153,14 +153,22 @@ def get_spent(conn, transaction_id, output): cursor = conn.run( conn.collection('bigchain').aggregate([ {'$match': { - 'block.transactions.inputs.fulfills.txid': transaction_id, - 'block.transactions.inputs.fulfills.output': output + 'block.transactions.inputs': { + '$elemMatch': { + 'fulfills.txid': transaction_id, + 'fulfills.output': output, + }, + }, }}, {'$unwind': '$block.transactions'}, {'$match': { - 'block.transactions.inputs.fulfills.txid': transaction_id, - 'block.transactions.inputs.fulfills.output': output - }} + 'block.transactions.inputs': { + '$elemMatch': { + 'fulfills.txid': transaction_id, + 'fulfills.output': output, + }, + }, + }}, ])) # we need to access some nested fields before returning so lets use a # generator to avoid having to read all records on the cursor at this point From 08f040d2186335e04560f97cc3ad844e74254319 Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 09:09:06 +0100 Subject: [PATCH 14/80] Authentication support --- bigchaindb/backend/connection.py | 6 ++++-- bigchaindb/backend/mongodb/connection.py | 27 +++++++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index cf6bece7..56b5cd82 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None, ssl=False): + connection_timeout=None, replicaset=None, ssl=False, login=None, password=None): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -52,6 +52,8 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, replicaset = replicaset or bigchaindb.config['database'].get('replicaset') ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ else ssl + login = login or bigchaindb.config['database'].get('login') + password = password or bigchaindb.config['database'].get('password') try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -65,7 +67,7 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, logger.debug('Connection: {}'.format(Class)) return Class(host=host, port=port, dbname=dbname, max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset, ssl=ssl) + replicaset=replicaset, ssl=ssl, login=login, password=password) class Connection: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 274d64c1..9168190a 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, ssl=False, **kwargs): + def __init__(self, replicaset=None, ssl=False, login=None, password=None, **kwargs): """Create a new Connection instance. Args: @@ -30,6 +30,8 @@ class MongoDBConnection(Connection): self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] self.ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ else ssl + self.login = login or bigchaindb.config['database'].get('login') + self.password = password or bigchaindb.config['database'].get('password') @property def db(self): @@ -73,15 +75,20 @@ class MongoDBConnection(Connection): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout, self.ssl) + initialize_replica_set(self.host, self.port, self.connection_timeout, self.dbname, self.ssl, self.login, self.password) # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. - return pymongo.MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout, - ssl=self.ssl) + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) + + if self.login is not None and self.password is not None: + client[self.dbname].authenticate(self.login, self.password) + + return client # `initialize_replica_set` might raise `ConnectionFailure` or `OperationFailure`. except (pymongo.errors.ConnectionFailure, @@ -89,7 +96,7 @@ class MongoDBConnection(Connection): raise ConnectionError() from exc -def initialize_replica_set(host, port, connection_timeout, ssl): +def initialize_replica_set(host, port, connection_timeout, dbname, ssl, login, password): """Initialize a replica set. If already initialized skip.""" # Setup a MongoDB connection @@ -100,6 +107,10 @@ def initialize_replica_set(host, port, connection_timeout, ssl): port=port, serverselectiontimeoutms=connection_timeout, ssl=ssl) + + if login is not None and password is not None: + conn[dbname].authenticate(login, password) + _check_replica_set(conn) host = '{}:{}'.format(bigchaindb.config['database']['host'], bigchaindb.config['database']['port']) From ff132a9464e60197289d95ea4474123c874807f6 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 17 Mar 2017 09:28:16 +0100 Subject: [PATCH 15/80] Updated copyright year in docs footers --- docs/root/source/conf.py | 2 +- docs/server/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py index 50dec3ea..0d799fed 100644 --- a/docs/root/source/conf.py +++ b/docs/root/source/conf.py @@ -58,7 +58,7 @@ master_doc = 'index' # General information about the project. project = 'BigchainDB' -copyright = '2016, BigchainDB Contributors' +copyright = '2017, BigchainDB Contributors' author = 'BigchainDB Contributors' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/server/source/conf.py b/docs/server/source/conf.py index 5550e994..756a8d13 100644 --- a/docs/server/source/conf.py +++ b/docs/server/source/conf.py @@ -82,7 +82,7 @@ master_doc = 'index' # General information about the project. project = 'BigchainDB Server' -copyright = '2016' +copyright = '2017, BigchainDB Contributors' author = 'BigchainDB Contributors' # The version info for the project you're documenting, acts as replacement for From 94877cb9c19f2a9c894b0c234f5c4cd3eaa05788 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 16 Mar 2017 17:01:01 +0100 Subject: [PATCH 16/80] Update change log --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 538d2ccc..9af7ccc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [0.9.4] - 2017-03-16 +Tag name: v0.9.4 + +### Fixed +Fixed #1271 (false double spend error). Thanks to @jmduque for reporting the +problem along with a very detailed diagnosis and useful recommendations. + ## [0.9.3] - 2017-03-06 Tag name: v0.9.3 From 8526246f7802141fc323540250672b9b248b88f6 Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 10:01:58 +0100 Subject: [PATCH 17/80] Fix unit test --- tests/backend/mongodb/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index 6350a7c5..a02b1735 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -168,7 +168,7 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] # check that it returns - assert initialize_replica_set('host', 1337, 1000) is None + assert initialize_replica_set('host', 1337, 1000, False, None, None) is None # test it raises OperationError if anything wrong with mock.patch.object(Database, 'command') as mock_command: @@ -178,4 +178,4 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] with pytest.raises(pymongo.errors.OperationFailure): - initialize_replica_set('host', 1337, 1000) + initialize_replica_set('host', 1337, 1000, False, None, None) From 3b1e6adb43b72d0b3ae1a13ffdaefcc6484f95cd Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 10:05:11 +0100 Subject: [PATCH 18/80] Formatting --- bigchaindb/backend/mongodb/connection.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 9168190a..8b30b2db 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -75,16 +75,17 @@ class MongoDBConnection(Connection): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout, self.dbname, self.ssl, self.login, self.password) + initialize_replica_set(self.host, self.port, self.connection_timeout, + self.dbname, self.ssl, self.login, self.password) # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. - client = pymongo.MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout, - ssl=self.ssl) - + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) + if self.login is not None and self.password is not None: client[self.dbname].authenticate(self.login, self.password) From 3b99daa0802d2999ccd845634987394cd9dac92e Mon Sep 17 00:00:00 2001 From: Andrej Svenke Date: Fri, 17 Mar 2017 12:18:20 +0100 Subject: [PATCH 19/80] Docker image optimization. (#1277) --- Dockerfile | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index bcfa8609..021f6772 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,32 @@ FROM ubuntu:xenial -# From http://stackoverflow.com/a/38553499 - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y locales - -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ - echo 'LANG="en_US.UTF-8"'>/etc/default/locale && \ - dpkg-reconfigure --frontend=noninteractive locales && \ - update-locale LANG=en_US.UTF-8 - ENV LANG en_US.UTF-8 - -# The `apt-get update` command executed with the install instructions should -# not use a locally cached storage layer. Force update the cache again. -# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -RUN apt-get update && apt-get -y install python3 python3-pip libffi-dev \ - && pip3 install --upgrade pip \ - && pip3 install --upgrade setuptools +ENV DEBIAN_FRONTEND noninteractive RUN mkdir -p /usr/src/app - COPY . /usr/src/app/ - WORKDIR /usr/src/app -RUN pip3 install --no-cache-dir -e . +RUN locale-gen en_US.UTF-8 && \ + apt-get -q update && \ + apt-get install -qy --no-install-recommends \ + python3 \ + python3-pip \ + libffi-dev \ + python3-dev \ + build-essential && \ + \ + pip3 install --upgrade --no-cache-dir pip setuptools && \ + \ + pip3 install --no-cache-dir -e . && \ + \ + apt-get remove -qy --purge gcc cpp binutils perl && \ + apt-get -qy autoremove && \ + apt-get -q clean all && \ + rm -rf /usr/share/perl /usr/share/perl5 /usr/share/man /usr/share/info /usr/share/doc && \ + rm -rf /var/lib/apt/lists/* VOLUME ["/data"] - WORKDIR /data ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb From 550b9cb804db8a6add84eea768b7b9355bbe27ac Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 10:33:26 +0100 Subject: [PATCH 20/80] Fix unit test --- tests/backend/mongodb/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index a02b1735..e0b161b0 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -168,7 +168,7 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] # check that it returns - assert initialize_replica_set('host', 1337, 1000, False, None, None) is None + assert initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None) is None # test it raises OperationError if anything wrong with mock.patch.object(Database, 'command') as mock_command: @@ -178,4 +178,4 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] with pytest.raises(pymongo.errors.OperationFailure): - initialize_replica_set('host', 1337, 1000, False, None, None) + initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None) From 10365bae52a6a937b38881f3786322cb5da73332 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 17 Mar 2017 13:40:06 +0100 Subject: [PATCH 21/80] docs: removed docs re load testing w/ Docker --- .../source/appendices/run-with-docker.md | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index 6700391e..516978dd 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -140,38 +140,6 @@ machine running the Docker engine. If you are running docker-machine (e.g. on Mac OS X) this will be the IP of the Docker machine (`docker-machine ip machine_name`). -### Load Testing with Docker - -Now that we have BigchainDB running in the Docker container named `bigchaindb`, we can -start another BigchainDB container to generate a load test for it. - -First, make sure the container named `bigchaindb` is still running. You can check that using: -```text -docker ps -``` - -You should see a container named `bigchaindb` in the list. - -You can load test the BigchainDB running in that container by running the `bigchaindb load` command in a second container: - -```text -docker run \ - --env BIGCHAINDB_DATABASE_HOST=bigchaindb \ - --link bigchaindb \ - --rm \ - --volume "$HOME/bigchaindb_docker:/data" \ - bigchaindb/bigchaindb \ - load -``` - -Note the `--link` option to link to the first container (named `bigchaindb`). - -Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](../server-reference/bigchaindb-cli.html). - -If you look at the RethinkDB dashboard (in your web browser), you should see the effects of the load test. You can also see some effects in the Docker logs using: -```text -docker logs -f bigchaindb -``` ## Building Your Own Image From 43f779a18b35af04db96079fcd72caa31dc60af9 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 13 Mar 2017 17:55:11 +0100 Subject: [PATCH 22/80] Add logging infrastructure --- bigchaindb/__init__.py | 17 +- bigchaindb/commands/bigchain.py | 4 + bigchaindb/log/__init__.py | 0 bigchaindb/log/configs.py | 59 ++++ bigchaindb/log/setup.py | 169 +++++++++++ setup.py | 1 + tests/commands/conftest.py | 9 + tests/commands/rethinkdb/test_commands.py | 4 +- tests/commands/test_commands.py | 39 ++- tests/log/test_setup.py | 328 ++++++++++++++++++++++ tests/test_config_utils.py | 3 +- tests/web/test_transactions.py | 74 ++++- 12 files changed, 680 insertions(+), 27 deletions(-) create mode 100644 bigchaindb/log/__init__.py create mode 100644 bigchaindb/log/configs.py create mode 100644 bigchaindb/log/setup.py create mode 100644 tests/log/test_setup.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 1df2551c..c0e4fd56 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -45,7 +45,22 @@ config = { 'private': None, }, 'keyring': [], - 'backlog_reassign_delay': 120 + 'backlog_reassign_delay': 120, + 'log': { + # TODO Document here or elsewhere. + # Example of config: + # 'file': '/var/log/bigchaindb.log', + # 'level_console': 'info', + # 'level_logfile': 'info', + # 'datefmt_console': '%Y-%m-%d %H:%M:%S', + # 'datefmt_logfile': '%Y-%m-%d %H:%M:%S', + # 'fmt_console': '%(asctime)s [%(levelname)s] (%(name)s) %(message)s', + # 'fmt_logfile': '%(asctime)s [%(levelname)s] (%(name)s) %(message)s', + # 'granular_levels': { + # 'bichaindb.backend': 'info', + # 'bichaindb.core': 'info', + # }, + }, } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 767f6ccc..62f3a7f6 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -25,6 +25,7 @@ from bigchaindb.commands.messages import ( RETHINKDB_STARTUP_ERROR, ) from bigchaindb.commands.utils import configure_bigchaindb, input_on_stderr +from bigchaindb.log.setup import setup_logging logging.basicConfig(level=logging.INFO) @@ -173,6 +174,9 @@ def run_start(args): """Start the processes to run the node""" logger.info('BigchainDB Version %s', bigchaindb.__version__) + # TODO setup logging -- pass logging config, extracted out from main config + setup_logging() + if args.allow_temp_keypair: if not (bigchaindb.config['keypair']['private'] or bigchaindb.config['keypair']['public']): diff --git a/bigchaindb/log/__init__.py b/bigchaindb/log/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bigchaindb/log/configs.py b/bigchaindb/log/configs.py new file mode 100644 index 00000000..7a8acc7c --- /dev/null +++ b/bigchaindb/log/configs.py @@ -0,0 +1,59 @@ +import logging +from os.path import expanduser, join + + +DEFAULT_LOG_DIR = expanduser('~') + +PUBLISHER_LOGGING_CONFIG = { + 'version': 1, + 'disable_existing_loggers': False, + 'root': { + 'level': logging.DEBUG, + }, +} + +SUBSCRIBER_LOGGING_CONFIG = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'console': { + 'class': 'logging.Formatter', + 'format': ( + '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s' + ), + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + 'file': { + 'class': 'logging.Formatter', + 'format': ('[%(asctime)s] [%(levelname)s] (%(name)s) ' + '%(message)s (%(processName)-10s - pid: %(process)d)'), + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'console', + 'level': logging.INFO, + }, + 'file': { + 'class': 'logging.FileHandler', + 'filename': join(DEFAULT_LOG_DIR, 'bigchaindb.log'), + 'mode': 'w', + 'formatter': 'file', + 'level': logging.INFO, + }, + 'errors': { + 'class': 'logging.FileHandler', + 'filename': join(DEFAULT_LOG_DIR, 'bigchaindb-errors.log'), + 'mode': 'w', + 'level': logging.ERROR, + 'formatter': 'file', + }, + }, + 'loggers': {}, + 'root': { + 'level': logging.DEBUG, + 'handlers': ['console', 'file', 'errors'] + }, +} diff --git a/bigchaindb/log/setup.py b/bigchaindb/log/setup.py new file mode 100644 index 00000000..fdf8e49b --- /dev/null +++ b/bigchaindb/log/setup.py @@ -0,0 +1,169 @@ +"""Setup logging.""" +from copy import deepcopy +import logging +from logging.config import dictConfig +import logging.handlers +import pickle +from socketserver import StreamRequestHandler, ThreadingTCPServer +import struct +import sys +from multiprocessing import Process + +from .configs import PUBLISHER_LOGGING_CONFIG, SUBSCRIBER_LOGGING_CONFIG +from bigchaindb.common.exceptions import ConfigurationError + + +def _normalize_log_level(level): + try: + return level.upper() + except AttributeError as exc: + raise ConfigurationError('Log level must be a string!') from exc + + +def setup_pub_logger(): + dictConfig(PUBLISHER_LOGGING_CONFIG) + socket_handler = logging.handlers.SocketHandler( + 'localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) + socket_handler.setLevel(logging.DEBUG) + logger = logging.getLogger() + logger.addHandler(socket_handler) + + +def setup_sub_logger(*, user_log_config=None): + server = LogRecordSocketServer() + with server: + server_proc = Process( + target=server.serve_forever, + kwargs={'log_config': user_log_config}, + ) + server_proc.start() + + +def setup_logging(*, user_log_config=None): + setup_pub_logger() + setup_sub_logger(user_log_config=user_log_config) + + +def create_subscriber_logging_config(*, user_log_config=None): + sub_log_config = deepcopy(SUBSCRIBER_LOGGING_CONFIG) + + if not user_log_config: + return sub_log_config + + if 'file' in user_log_config: + filename = user_log_config['file'] + sub_log_config['handlers']['file']['filename'] = filename + + if 'level_console' in user_log_config: + level = _normalize_log_level(user_log_config['level_console']) + sub_log_config['handlers']['console']['level'] = level + + if 'level_logfile' in user_log_config: + level = _normalize_log_level(user_log_config['level_logfile']) + sub_log_config['handlers']['file']['level'] = level + + if 'fmt_console' in user_log_config: + fmt = user_log_config['fmt_console'] + sub_log_config['formatters']['console']['format'] = fmt + + if 'fmt_logfile' in user_log_config: + fmt = user_log_config['fmt_logfile'] + sub_log_config['formatters']['file']['format'] = fmt + + if 'datefmt_console' in user_log_config: + fmt = user_log_config['datefmt_console'] + sub_log_config['formatters']['console']['datefmt'] = fmt + + if 'datefmt_logfile' in user_log_config: + fmt = user_log_config['datefmt_logfile'] + sub_log_config['formatters']['file']['datefmt'] = fmt + + log_levels = user_log_config.get('granular_levels', {}) + + for logger_name, level in log_levels.items(): + level = _normalize_log_level(level) + try: + sub_log_config['loggers'][logger_name]['level'] = level + except KeyError: + sub_log_config['loggers'][logger_name] = {'level': level} + + return sub_log_config + + +class LogRecordStreamHandler(StreamRequestHandler): + """Handler for a streaming logging request. + + This basically logs the record using whatever logging policy is + configured locally. + """ + + def handle(self): + """ + Handle multiple requests - each expected to be a 4-byte length, + followed by the LogRecord in pickle format. Logs the record + according to whatever policy is configured locally. + """ + while True: + chunk = self.connection.recv(4) + if len(chunk) < 4: + break + slen = struct.unpack('>L', chunk)[0] + chunk = self.connection.recv(slen) + while len(chunk) < slen: + chunk = chunk + self.connection.recv(slen - len(chunk)) + obj = self.unpickle(chunk) + record = logging.makeLogRecord(obj) + self.handle_log_record(record) + + def unpickle(self, data): + try: + return pickle.loads(data) + except (pickle.UnpicklingError, + AttributeError, EOFError, TypeError) as exc: + return { + 'msg': '({}) Log handling error: un-pickling failed!'.format( + exc.__class__.__name__), + 'exc_info': exc.args, + 'level': logging.ERROR, + 'func': self.unpickle.__name__, + } + + def handle_log_record(self, record): + logger = logging.getLogger(record.name) + logger.handle(record) + + +class LogRecordSocketServer(ThreadingTCPServer): + """ + Simple TCP socket-based logging server. + + """ + allow_reuse_address = True + + def __init__(self, + host='localhost', + port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, + handler=LogRecordStreamHandler): + super().__init__((host, port), handler) + + def serve_forever(self, *, poll_interval=0.5, log_config=None): + sub_logging_config = create_subscriber_logging_config( + user_log_config=log_config) + dictConfig(sub_logging_config) + try: + super().serve_forever(poll_interval=poll_interval) + except KeyboardInterrupt: + pass + + +# NOTE: Because the context manager is only available +# from 3.6 and up, we add it for lower versions. +if sys.version_info < (3, 6): + def __enter__(self): + return self + + def __exit__(self, *args): + self.server_close() + + LogRecordSocketServer.__enter__ = __enter__ + LogRecordSocketServer.__exit__ = __exit__ diff --git a/setup.py b/setup.py index dadd7385..7a38bb1f 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ dev_require = [ 'ipdb', 'ipython', 'watchdog', + 'logging_tree', ] docs_require = [ diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index fde478b5..96a2c608 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -50,3 +50,12 @@ def run_start_args(request): start_rethinkdb=param.get('start_rethinkdb', False), allow_temp_keypair=param.get('allow_temp_keypair', False), ) + + +@pytest.fixture +def mocked_setup_logging(mocker): + return mocker.patch( + 'bigchaindb.commands.bigchain.setup_logging', + autospec=True, + spec_set=True, + ) diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index f0ae1090..bf6e0931 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -9,12 +9,14 @@ from argparse import Namespace def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, mock_run_configure, mock_processes_start, - mock_db_init_with_existing_db): + mock_db_init_with_existing_db, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_start args = Namespace(start_rethinkdb=True, allow_temp_keypair=False, config=None, yes=True) run_start(args) mock_start_rethinkdb.assert_called_with() + mocked_setup_logging.assert_called_once_with() @patch('subprocess.Popen') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 198c39d1..8bf00959 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -33,15 +33,20 @@ def test_main_entrypoint(mock_start): assert mock_start.called -def test_bigchain_run_start(mock_run_configure, mock_processes_start, mock_db_init_with_existing_db): +def test_bigchain_run_start(mock_run_configure, + mock_processes_start, + mock_db_init_with_existing_db, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_start args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True) run_start(args) + mocked_setup_logging.assert_called_once_with() @pytest.mark.skipif(reason="BigchainDB doesn't support the automatic creation of a config file anymore") -def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_processes_start, - mock_generate_key_pair, mock_db_init_with_existing_db): +def test_bigchain_run_start_assume_yes_create_default_config( + monkeypatch, mock_processes_start, mock_generate_key_pair, + mock_db_init_with_existing_db, mocked_setup_logging): import bigchaindb from bigchaindb.commands.bigchain import run_start from bigchaindb import config_utils @@ -61,6 +66,7 @@ def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_p args = Namespace(config=None, yes=True) run_start(args) + mocked_setup_logging.assert_called_once_with() assert value['return'] == expected_config @@ -228,9 +234,9 @@ def test_run_configure_with_backend(backend, monkeypatch, mock_write_config): @patch('bigchaindb.common.crypto.generate_key_pair', return_value=('private_key', 'public_key')) @pytest.mark.usefixtures('ignore_local_config_file') -def test_allow_temp_keypair_generates_one_on_the_fly(mock_gen_keypair, - mock_processes_start, - mock_db_init_with_existing_db): +def test_allow_temp_keypair_generates_one_on_the_fly( + mock_gen_keypair, mock_processes_start, + mock_db_init_with_existing_db, mocked_setup_logging): import bigchaindb from bigchaindb.commands.bigchain import run_start @@ -239,6 +245,7 @@ def test_allow_temp_keypair_generates_one_on_the_fly(mock_gen_keypair, args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True) run_start(args) + mocked_setup_logging.assert_called_once_with() assert bigchaindb.config['keypair']['private'] == 'private_key' assert bigchaindb.config['keypair']['public'] == 'public_key' @@ -248,7 +255,8 @@ def test_allow_temp_keypair_generates_one_on_the_fly(mock_gen_keypair, @pytest.mark.usefixtures('ignore_local_config_file') def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, mock_processes_start, - mock_db_init_with_existing_db): + mock_db_init_with_existing_db, + mocked_setup_logging): import bigchaindb from bigchaindb.commands.bigchain import run_start @@ -262,11 +270,15 @@ def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True) run_start(args) + mocked_setup_logging.assert_called_once_with() assert bigchaindb.config['keypair']['private'] == original_private_key assert bigchaindb.config['keypair']['public'] == original_public_key -def test_run_start_when_db_already_exists(mocker, monkeypatch, run_start_args): +def test_run_start_when_db_already_exists(mocker, + monkeypatch, + run_start_args, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_start from bigchaindb.common.exceptions import DatabaseAlreadyExists mocked_start = mocker.patch('bigchaindb.processes.start') @@ -277,10 +289,14 @@ def test_run_start_when_db_already_exists(mocker, monkeypatch, run_start_args): monkeypatch.setattr( 'bigchaindb.commands.bigchain._run_init', mock_run_init) run_start(run_start_args) + mocked_setup_logging.assert_called_once_with() assert mocked_start.called -def test_run_start_when_keypair_not_found(mocker, monkeypatch, run_start_args): +def test_run_start_when_keypair_not_found(mocker, + monkeypatch, + run_start_args, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_start from bigchaindb.commands.messages import CANNOT_START_KEYPAIR_NOT_FOUND from bigchaindb.common.exceptions import KeypairNotFoundException @@ -295,6 +311,7 @@ def test_run_start_when_keypair_not_found(mocker, monkeypatch, run_start_args): with pytest.raises(SystemExit) as exc: run_start(run_start_args) + mocked_setup_logging.assert_called_once_with() assert len(exc.value.args) == 1 assert exc.value.args[0] == CANNOT_START_KEYPAIR_NOT_FOUND assert not mocked_start.called @@ -302,7 +319,8 @@ def test_run_start_when_keypair_not_found(mocker, monkeypatch, run_start_args): def test_run_start_when_start_rethinkdb_fails(mocker, monkeypatch, - run_start_args): + run_start_args, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_start from bigchaindb.commands.messages import RETHINKDB_STARTUP_ERROR from bigchaindb.common.exceptions import StartupError @@ -319,6 +337,7 @@ def test_run_start_when_start_rethinkdb_fails(mocker, with pytest.raises(SystemExit) as exc: run_start(run_start_args) + mocked_setup_logging.assert_called_once_with() assert len(exc.value.args) == 1 assert exc.value.args[0] == RETHINKDB_STARTUP_ERROR.format(err_msg) assert not mocked_start.called diff --git a/tests/log/test_setup.py b/tests/log/test_setup.py new file mode 100644 index 00000000..e0434eb9 --- /dev/null +++ b/tests/log/test_setup.py @@ -0,0 +1,328 @@ +import logging +import pickle +from logging import getLogger +from logging.config import dictConfig +from logging.handlers import SocketHandler + +from pytest import fixture, mark, raises + + +@fixture +def reset_logging_config(): + original_root_logger_level = getLogger().level + dictConfig({'version': 1, 'root': {'level': 'NOTSET'}}) + yield + getLogger().setLevel(original_root_logger_level) + + +@fixture +def mocked_process(mocker): + return mocker.patch( + 'bigchaindb.log.setup.Process', autospec=True, spec_set=True) + + +@fixture +def mocked_socket_server(mocker): + return mocker.patch( + 'bigchaindb.log.setup.LogRecordSocketServer', + autospec=True, + spec_set=True, + ) + + +@fixture +def mocked_setup_pub_logger(mocker): + return mocker.patch( + 'bigchaindb.log.setup.setup_pub_logger', autospec=True, spec_set=True) + + +@fixture +def mocked_setup_sub_logger(mocker): + return mocker.patch( + 'bigchaindb.log.setup.setup_sub_logger', autospec=True, spec_set=True) + + +@fixture +def log_record_dict(): + return { + 'args': None, + 'created': 1489584900.595193, + 'exc_info': None, + 'exc_text': None, + 'filename': 'config_utils.py', + 'funcName': 'autoconfigure', + 'levelname': 'DEBUG', + 'levelno': 10, + 'lineno': 228, + 'module': 'config_utils', + 'msecs': 595.1929092407227, + 'msg': 'System already configured, skipping autoconfiguration', + 'name': 'bigchaindb.config_utils', + 'pathname': '/usr/src/app/bigchaindb/config_utils.py', + 'process': 1981, + 'processName': 'MainProcess', + 'relativeCreated': 398.4854221343994, + 'stack_info': None, + 'thread': 140352503879424, + 'threadName': 'MainThread', + } + + +@fixture +def log_record(log_record_dict): + return logging.makeLogRecord(log_record_dict) + + +@fixture +def log_record_bytes(log_record_dict): + return pickle.dumps(log_record_dict) + + +@mark.usefixtures('reset_logging_config') +def test_setup_logging(mocked_setup_pub_logger, mocked_setup_sub_logger): + from bigchaindb.log.setup import setup_logging + setup_logging() + mocked_setup_pub_logger.assert_called_once_with() + mocked_setup_sub_logger.assert_called_once_with(user_log_config=None) + + +@mark.usefixtures('reset_logging_config') +def test_setup_pub_logger(): + from bigchaindb.log.setup import setup_pub_logger + from bigchaindb.log.configs import PUBLISHER_LOGGING_CONFIG + root_logger = getLogger() + assert root_logger.level == logging.NOTSET + setup_pub_logger() + assert root_logger.level == PUBLISHER_LOGGING_CONFIG['root']['level'] + assert root_logger.hasHandlers() + assert isinstance(root_logger.handlers[0], SocketHandler) + + +@mark.usefixtures('reset_logging_config') +def test_setup_sub_logger_without_config(mocked_socket_server, mocked_process): + from bigchaindb.log.setup import setup_sub_logger + setup_sub_logger() + root_logger = getLogger() + assert root_logger.level == logging.NOTSET + mocked_socket_server.assert_called_once_with() + mocked_process.assert_called_once_with( + target=mocked_socket_server.return_value.serve_forever, + kwargs={'log_config': None}, + ) + mocked_process.return_value.start.assert_called_once_with() + + +@mark.usefixtures('reset_logging_config') +def test_setup_sub_logger_with_config(mocked_socket_server, mocked_process): + from bigchaindb.log.setup import setup_sub_logger + user_log_config = { + 'file': '/var/log/bdb.log', + 'level_console': 'warning', + 'level_logfile': 'info', + 'fmt_console': '[%(levelname)s] (%(name)s) %(message)s', + 'fmt_logfile': '[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s', + 'granular_levels': { + 'bigchaindb.core': 'debug', + }, + } + root_logger = getLogger() + setup_sub_logger(user_log_config=user_log_config) + assert root_logger.level == logging.NOTSET + mocked_socket_server.assert_called_once_with() + mocked_process.assert_called_once_with( + target=mocked_socket_server.return_value.serve_forever, + kwargs={'log_config': user_log_config}, + ) + mocked_process.return_value.start.assert_called_once_with() + + +def test_create_subscriber_logging_config_without_user_given_config(): + from bigchaindb.log.setup import create_subscriber_logging_config + from bigchaindb.log.configs import SUBSCRIBER_LOGGING_CONFIG + config = create_subscriber_logging_config() + assert config == SUBSCRIBER_LOGGING_CONFIG + + +def test_create_subscriber_logging_config_with_user_given_config(): + from bigchaindb.log.setup import create_subscriber_logging_config + from bigchaindb.log.configs import ( + SUBSCRIBER_LOGGING_CONFIG as expected_log_config) + user_log_config = { + 'file': '/var/log/bigchaindb/bdb.log', + 'level_console': 'warning', + 'level_logfile': 'info', + 'fmt_console': '[%(levelname)s] (%(name)s) %(message)s', + 'fmt_logfile': '[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s', + 'datefmt_console': '%H:%M:%S', + 'datefmt_logfile': '%a, %d %b %Y %H:%M:%S +0000', + 'granular_levels': { + 'bigchaindb.core': 'debug', + }, + } + config = create_subscriber_logging_config(user_log_config=user_log_config) + assert config['root']['level'] == expected_log_config['root']['level'] + assert all(config['loggers'][logger]['level'] == level.upper() + for logger, level in user_log_config['granular_levels'].items()) + assert len(config) == len(expected_log_config) + assert config['version'] == expected_log_config['version'] + assert (config['disable_existing_loggers'] == + expected_log_config['disable_existing_loggers']) + assert (config['formatters']['console']['format'] == + user_log_config['fmt_console']) + assert (config['formatters']['file']['format'] == + user_log_config['fmt_logfile']) + assert (config['formatters']['console']['datefmt'] == + user_log_config['datefmt_console']) + assert (config['formatters']['file']['datefmt'] == + user_log_config['datefmt_logfile']) + assert (config['handlers']['console']['level'] == + user_log_config['level_console'].upper()) + assert (config['handlers']['file']['level'] == + user_log_config['level_logfile'].upper()) + assert config['handlers']['file']['filename'] == user_log_config['file'] + del config['handlers']['console']['level'] + del config['handlers']['file']['level'] + del config['handlers']['file']['filename'] + del config['formatters']['console']['format'] + del config['formatters']['console']['datefmt'] + del config['formatters']['file']['format'] + del config['formatters']['file']['datefmt'] + del expected_log_config['handlers']['console']['level'] + del expected_log_config['handlers']['file']['level'] + del expected_log_config['handlers']['file']['filename'] + del expected_log_config['formatters']['console']['format'] + del expected_log_config['formatters']['console']['datefmt'] + del expected_log_config['formatters']['file']['format'] + del expected_log_config['formatters']['file']['datefmt'] + assert (config['handlers']['console'] == + expected_log_config['handlers']['console']) + assert (config['handlers']['file'] == + expected_log_config['handlers']['file']) + assert (config['formatters']['console'] == + expected_log_config['formatters']['console']) + assert (config['formatters']['file'] == + expected_log_config['formatters']['file']) + + +def test_normalize_log_level(): + from bigchaindb.common.exceptions import ConfigurationError + from bigchaindb.log.setup import _normalize_log_level + with raises(ConfigurationError) as exc: + _normalize_log_level(2) + assert exc.value.args == ('Log level must be a string!',) + assert isinstance(exc.value.__cause__, AttributeError) + assert exc.value.__cause__.args == ( + "'int' object has no attribute 'upper'",) + + +class TestLogRecordSocketServer: + + def test_init(self): + from bigchaindb.log.setup import (LogRecordSocketServer, + LogRecordStreamHandler) + server = LogRecordSocketServer() + assert server.allow_reuse_address + assert server.server_address == ( + '127.0.0.1', logging.handlers.DEFAULT_TCP_LOGGING_PORT) + assert server.RequestHandlerClass == LogRecordStreamHandler + + @mark.parametrize('side_effect', (None, KeyboardInterrupt)) + def test_server_forever(self, mocker, side_effect): + from bigchaindb.log.setup import LogRecordSocketServer + nocked_create_subscriber_logging_config = mocker.patch( + 'bigchaindb.log.setup.create_subscriber_logging_config', + autospec=True, + spec_set=True, + ) + mocked_dict_config = mocker.patch('bigchaindb.log.setup.dictConfig', + autospec=True, spec_set=True) + mocked_parent_serve_forever = mocker.patch( + 'bigchaindb.log.setup.ThreadingTCPServer.serve_forever', + autospec=True, + spec_set=True, + side_effect=side_effect, + ) + server = LogRecordSocketServer() + with server: + server.serve_forever() + nocked_create_subscriber_logging_config.assert_called_once_with( + user_log_config=None) + mocked_dict_config.assert_called_once_with( + nocked_create_subscriber_logging_config.return_value) + mocked_parent_serve_forever.assert_called_once_with(server, + poll_interval=0.5) + + +class TestLogRecordStreamHandler: + + def test_handle(self, mocker, log_record_dict, log_record_bytes): + from bigchaindb.log.setup import LogRecordStreamHandler + + chunks = [log_record_bytes, b'\x00\x00\x02T'] + mocked_handle_log_record = mocker.patch( + 'bigchaindb.log.setup.LogRecordStreamHandler.handle_log_record', + autospec=True, + spec_set=True, + ) + + def mocked_recv(bufsize): + try: + return chunks.pop() + except IndexError: + return b' ' + + request = mocker.patch('socket.socket', autospec=True, spec_set=True) + request.return_value.recv = mocked_recv + client_address = ('127.0.0.1', 9020) + LogRecordStreamHandler( + request.return_value, client_address, None) + assert mocked_handle_log_record.called + assert (mocked_handle_log_record.call_args[0][1].__dict__ == + log_record_dict) + + def test_handle_log_record(self, mocker, log_record): + from bigchaindb.log.setup import LogRecordStreamHandler + mocker.patch('bigchaindb.log.setup.LogRecordStreamHandler.handle') + mocked_logger_handle = mocker.patch( + 'bigchaindb.log.setup.logging.Logger.handle', + autospec=True, spec_set=True) + request = mocker.patch('socket.socket', autospec=True, spec_set=True) + client_address = ('127.0.0.1', 9020) + handler = LogRecordStreamHandler( + request.return_value, client_address, None) + handler.handle_log_record(log_record) + assert log_record in mocked_logger_handle.call_args[0] + + def test_unpickle(self, mocker, log_record_bytes, log_record_dict): + from bigchaindb.log.setup import LogRecordStreamHandler + mocker.patch('bigchaindb.log.setup.LogRecordStreamHandler.handle') + request = mocker.patch('socket.socket', autospec=True, spec_set=True) + client_address = ('127.0.0.1', 9020) + handler = LogRecordStreamHandler( + request.return_value, client_address, None) + obj = handler.unpickle(log_record_bytes) + assert obj == log_record_dict + + @mark.parametrize('error', ( + pickle.UnpicklingError, AttributeError, EOFError, TypeError)) + def test_unpickle_error(self, mocker, error): + from bigchaindb.log.setup import LogRecordStreamHandler + mocker.patch('bigchaindb.log.setup.LogRecordStreamHandler.handle') + mocker.patch( + 'bigchaindb.log.setup.pickle.loads', + autospec=True, + spec_set=True, + side_effect=error('msg'), + ) + request = mocker.patch('socket.socket', autospec=True, spec_set=True) + client_address = ('127.0.0.1', 9020) + handler = LogRecordStreamHandler( + request.return_value, client_address, None) + obj = handler.unpickle(None) + assert obj == { + 'msg': '({}) Log handling error: un-pickling failed!'.format( + error.__name__), + 'exc_info': ('msg',), + 'level': logging.ERROR, + 'func': handler.unpickle.__name__, + } diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 0fa5135b..4234e242 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -202,7 +202,8 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'private': None, }, 'keyring': KEYRING.split(':'), - 'backlog_reassign_delay': 5 + 'backlog_reassign_delay': 5, + 'log': {}, } diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 5533dbd0..4c6e76c1 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -44,7 +44,8 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub -def test_post_create_transaction_with_invalid_id(b, client, caplog): +@patch('bigchaindb.web.views.base.logger') +def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() @@ -56,16 +57,29 @@ def test_post_create_transaction_with_invalid_id(b, client, caplog): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) expected_status_code = 400 expected_error_message = ( - 'Invalid transaction ({}): The transaction\'s id \'{}\' isn\'t equal to ' - 'the hash of its body, i.e. it\'s not valid.' + "Invalid transaction ({}): The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid." ).format(InvalidHash.__name__, tx['id']) assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message - assert caplog.records[0].args['status'] == expected_status_code - assert caplog.records[0].args['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + {'message': expected_error_message, 'status': expected_status_code} in + mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[0].args['status'] == expected_status_code + # assert caplog.records[0].args['message'] == expected_error_message -def test_post_create_transaction_with_invalid_signature(b, client, caplog): +@patch('bigchaindb.web.views.base.logger') +def test_post_create_transaction_with_invalid_signature(mock_logger, + b, + client): from bigchaindb.common.exceptions import InvalidSignature from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() @@ -82,8 +96,18 @@ def test_post_create_transaction_with_invalid_signature(b, client, caplog): ).format(InvalidSignature.__name__) assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message - assert caplog.records[0].args['status'] == expected_status_code - assert caplog.records[0].args['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + {'message': expected_error_message, 'status': expected_status_code} in + mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[0].args['status'] == expected_status_code + # assert caplog.records[0].args['message'] == expected_error_message def test_post_create_transaction_with_invalid_structure(client): @@ -91,7 +115,8 @@ def test_post_create_transaction_with_invalid_structure(client): assert res.status_code == 400 -def test_post_create_transaction_with_invalid_schema(client, caplog): +@patch('bigchaindb.web.views.base.logger') +def test_post_create_transaction_with_invalid_schema(mock_logger, client): from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() tx = Transaction.create( @@ -103,8 +128,18 @@ def test_post_create_transaction_with_invalid_schema(client, caplog): "Invalid transaction schema: 'version' is a required property") assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message - assert caplog.records[0].args['status'] == expected_status_code - assert caplog.records[0].args['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + {'message': expected_error_message, 'status': expected_status_code} in + mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[0].args['status'] == expected_status_code + # assert caplog.records[0].args['message'] == expected_error_message @pytest.mark.parametrize('exc,msg', ( @@ -118,7 +153,8 @@ def test_post_create_transaction_with_invalid_schema(client, caplog): ('TransactionNotInValidBlock', 'Wait, maybe?'), ('ValidationError', '?'), )) -def test_post_invalid_transaction(client, exc, msg, monkeypatch, caplog): +@patch('bigchaindb.web.views.base.logger') +def test_post_invalid_transaction(mock_logger, client, exc, msg, monkeypatch,): from bigchaindb.common import exceptions exc_cls = getattr(exceptions, exc) @@ -135,8 +171,18 @@ def test_post_invalid_transaction(client, exc, msg, monkeypatch, caplog): assert res.status_code == expected_status_code assert (res.json['message'] == 'Invalid transaction ({}): {}'.format(exc, msg)) - assert caplog.records[2].args['status'] == expected_status_code - assert caplog.records[2].args['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + {'message': expected_error_message, 'status': expected_status_code} in + mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[2].args['status'] == expected_status_code + # assert caplog.records[2].args['message'] == expected_error_message @pytest.mark.bdb From c0498abed395412afd7069ab17790c8d54c64716 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Fri, 24 Feb 2017 13:31:07 +0100 Subject: [PATCH 23/80] Add log-level option for all CLI commands --- bigchaindb/commands/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index b04499d9..adf0d19c 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -151,6 +151,10 @@ base_parser.add_argument('-c', '--config', help='Specify the location of the configuration file ' '(use "-" for stdout)') +base_parser.add_argument('-l', '--log-level', + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + help='Log level') + base_parser.add_argument('-y', '--yes', '--yes-please', action='store_true', help='Assume "yes" as answer to all prompts and run ' From d688e695e6f54e37bb5ea07266c4849d3575eb7e Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Fri, 24 Feb 2017 13:31:49 +0100 Subject: [PATCH 24/80] Configure logging in `bigchaindb_configure` decorator --- bigchaindb/commands/bigchain.py | 1 - bigchaindb/commands/utils.py | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 62f3a7f6..080a8cb2 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -25,7 +25,6 @@ from bigchaindb.commands.messages import ( RETHINKDB_STARTUP_ERROR, ) from bigchaindb.commands.utils import configure_bigchaindb, input_on_stderr -from bigchaindb.log.setup import setup_logging logging.basicConfig(level=logging.INFO) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index adf0d19c..aaa92804 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -16,6 +16,7 @@ import bigchaindb import bigchaindb.config_utils from bigchaindb import backend from bigchaindb.common.exceptions import StartupError +from bigchaindb.log.setup import setup_logging from bigchaindb.version import __version__ @@ -23,6 +24,12 @@ def configure_bigchaindb(command): @functools.wraps(command) def configure(args): bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + + logging_config = bigchaindb.config['logging'] or {} + if 'log_level' in args and args.log_level: + logging_config['level'] = args.log_level + setup_logging(logging_config) + command(args) return configure From f549b008138388b5caed2964b6f82867da76fcd7 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Fri, 24 Feb 2017 14:48:22 +0100 Subject: [PATCH 25/80] Add tests for setting log-level from CLI --- bigchaindb/config_utils.py | 7 ++++-- tests/commands/test_utils.py | 47 +++++++++++++++++++++++++++++++++--- tests/conftest.py | 11 +++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 87a25d3f..5a72a7d6 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -220,11 +220,14 @@ def write_config(config, filename=None): json.dump(config, f, indent=4) +def is_configured(): + return bool(bigchaindb.config.get('CONFIGURED')) + + def autoconfigure(filename=None, config=None, force=False): """Run ``file_config`` and ``env_config`` if the module has not been initialized.""" - - if not force and bigchaindb.config.get('CONFIGURED'): + if not force and is_configured(): logger.debug('System already configured, skipping autoconfiguration') return diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index aadd24b5..6879e0eb 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -1,9 +1,50 @@ import argparse import pytest +from argparse import ArgumentTypeError, Namespace from unittest.mock import patch +@pytest.fixture +def reset_bigchaindb_config(monkeypatch): + import bigchaindb + monkeypatch.setattr('bigchaindb.config', bigchaindb._config) + + +@pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config') +def test_configure_bigchaindb_configures_bigchaindb(): + from bigchaindb.commands.utils import configure_bigchaindb + from bigchaindb.config_utils import is_configured + assert not is_configured() + + @configure_bigchaindb + def test_configure(args): + assert is_configured() + + args = Namespace(config=None) + test_configure(args) + + +@pytest.mark.usefixtures('ignore_local_config_file', + 'reset_bigchaindb_config', + 'reset_logging_config') +@pytest.mark.parametrize('log_level', ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')) +def test_configure_bigchaindb_configures_logging(log_level): + import logging + from logging import getLogger + from bigchaindb.commands.utils import configure_bigchaindb + root_logger = getLogger() + assert root_logger.level == 0 + + @configure_bigchaindb + def test_configure_logger(args): + root_logger = getLogger() + assert root_logger.level == getattr(logging, log_level) + + args = Namespace(config=None, log_level=log_level) + test_configure_logger(args) + + def test_start_raises_if_command_not_implemented(): from bigchaindb.commands import utils from bigchaindb.commands.bigchain import create_parser @@ -51,13 +92,13 @@ def test_mongodb_host_type(): from bigchaindb.commands.utils import mongodb_host # bad port provided - with pytest.raises(argparse.ArgumentTypeError): + with pytest.raises(ArgumentTypeError): mongodb_host('localhost:11111111111') # no port information provided - with pytest.raises(argparse.ArgumentTypeError): + with pytest.raises(ArgumentTypeError): mongodb_host('localhost') # bad host provided - with pytest.raises(argparse.ArgumentTypeError): + with pytest.raises(ArgumentTypeError): mongodb_host(':27017') diff --git a/tests/conftest.py b/tests/conftest.py index e943d0a9..210d526e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,8 @@ import random import pytest +from logging import getLogger +from logging.config import dictConfig from bigchaindb.common import crypto TEST_DB_NAME = 'bigchain_test' @@ -203,6 +205,15 @@ def ignore_local_config_file(monkeypatch): mock_file_config) +@pytest.fixture +def reset_logging_config(): + # root_logger_level = getLogger().level + root_logger_level = 'DEBUG' + dictConfig({'version': 1, 'root': {'level': 'NOTSET'}}) + yield + getLogger().setLevel(root_logger_level) + + @pytest.fixture def user_sk(): return USER_PRIVATE_KEY From 53d5232be8787ac1c01b40dbb0e47249c20e1958 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 20 Mar 2017 15:46:54 +0100 Subject: [PATCH 26/80] Remove leftover line from rebase --- bigchaindb/commands/bigchain.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 080a8cb2..767f6ccc 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -173,9 +173,6 @@ def run_start(args): """Start the processes to run the node""" logger.info('BigchainDB Version %s', bigchaindb.__version__) - # TODO setup logging -- pass logging config, extracted out from main config - setup_logging() - if args.allow_temp_keypair: if not (bigchaindb.config['keypair']['private'] or bigchaindb.config['keypair']['public']): From eff8e3adf3f6a5a1a8ace24c7f26112a48fdd299 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 20 Mar 2017 15:47:47 +0100 Subject: [PATCH 27/80] Update logging related code and tests after rebase --- bigchaindb/commands/utils.py | 6 +-- tests/commands/conftest.py | 2 +- tests/commands/rethinkdb/test_commands.py | 18 +++++--- tests/commands/test_commands.py | 54 ++++++++++++++++------- tests/commands/test_utils.py | 11 +++-- tests/conftest.py | 12 +++++ tests/log/test_setup.py | 13 +----- 7 files changed, 77 insertions(+), 39 deletions(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index aaa92804..73313f05 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -25,10 +25,10 @@ def configure_bigchaindb(command): def configure(args): bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) - logging_config = bigchaindb.config['logging'] or {} + logging_config = bigchaindb.config['log'] or {} if 'log_level' in args and args.log_level: - logging_config['level'] = args.log_level - setup_logging(logging_config) + logging_config['level_console'] = args.log_level + setup_logging(user_log_config=logging_config) command(args) diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index 96a2c608..30c577f5 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -55,7 +55,7 @@ def run_start_args(request): @pytest.fixture def mocked_setup_logging(mocker): return mocker.patch( - 'bigchaindb.commands.bigchain.setup_logging', + 'bigchaindb.commands.utils.setup_logging', autospec=True, spec_set=True, ) diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index bf6e0931..ac100075 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -16,7 +16,7 @@ def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, run_start(args) mock_start_rethinkdb.assert_called_with() - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('subprocess.Popen') @@ -38,7 +38,7 @@ def test_start_rethinkdb_exits_when_cannot_start(mock_popen): @patch('rethinkdb.ast.Table.reconfigure') -def test_set_shards(mock_reconfigure, monkeypatch, b): +def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): from bigchaindb.commands.bigchain import run_set_shards # this will mock the call to retrieve the database config @@ -50,6 +50,8 @@ def test_set_shards(mock_reconfigure, monkeypatch, b): args = Namespace(num_shards=3, config=None) run_set_shards(args) mock_reconfigure.assert_called_with(replicas=1, shards=3, dry_run=False) + mocked_setup_logging.assert_called_once_with(user_log_config={}) + mocked_setup_logging.reset_mock() # this will mock the call to retrieve the database config # we will set it to return three replica @@ -59,9 +61,10 @@ def test_set_shards(mock_reconfigure, monkeypatch, b): monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_replicas) run_set_shards(args) mock_reconfigure.assert_called_with(replicas=3, shards=3, dry_run=False) + mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_set_shards_raises_exception(monkeypatch, b): +def test_set_shards_raises_exception(monkeypatch, b, mocked_setup_logging): from bigchaindb.commands.bigchain import run_set_shards # test that we are correctly catching the exception @@ -78,10 +81,11 @@ def test_set_shards_raises_exception(monkeypatch, b): with pytest.raises(SystemExit) as exc: run_set_shards(args) assert exc.value.args == ('Failed to reconfigure tables.',) + mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('rethinkdb.ast.Table.reconfigure') -def test_set_replicas(mock_reconfigure, monkeypatch, b): +def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): from bigchaindb.commands.bigchain import run_set_replicas # this will mock the call to retrieve the database config @@ -93,6 +97,8 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b): args = Namespace(num_replicas=2, config=None) run_set_replicas(args) mock_reconfigure.assert_called_with(replicas=2, shards=2, dry_run=False) + mocked_setup_logging.assert_called_once_with(user_log_config={}) + mocked_setup_logging.reset_mock() # this will mock the call to retrieve the database config # we will set it to return three shards @@ -102,9 +108,10 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b): monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_shards) run_set_replicas(args) mock_reconfigure.assert_called_with(replicas=2, shards=3, dry_run=False) + mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_set_replicas_raises_exception(monkeypatch, b): +def test_set_replicas_raises_exception(monkeypatch, b, mocked_setup_logging): from bigchaindb.commands.bigchain import run_set_replicas # test that we are correctly catching the exception @@ -121,3 +128,4 @@ def test_set_replicas_raises_exception(monkeypatch, b): with pytest.raises(SystemExit) as exc: run_set_replicas(args) assert exc.value.args == ('Failed to reconfigure tables.',) + mocked_setup_logging.assert_called_once_with(user_log_config={}) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 8bf00959..eebd86ea 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -40,7 +40,7 @@ def test_bigchain_run_start(mock_run_configure, from bigchaindb.commands.bigchain import run_start args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True) run_start(args) - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) @pytest.mark.skipif(reason="BigchainDB doesn't support the automatic creation of a config file anymore") @@ -74,7 +74,7 @@ def test_bigchain_run_start_assume_yes_create_default_config( # interfere with capsys. # See related issue: https://github.com/pytest-dev/pytest/issues/128 @pytest.mark.usefixtures('ignore_local_config_file') -def test_bigchain_show_config(capsys): +def test_bigchain_show_config(capsys, mocked_setup_logging): from bigchaindb import config from bigchaindb.commands.bigchain import run_show_config @@ -85,9 +85,11 @@ def test_bigchain_show_config(capsys): del config['CONFIGURED'] config['keypair']['private'] = 'x' * 45 assert output_config == config + mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): +def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch, + mocked_setup_logging): from bigchaindb import config from bigchaindb.commands.bigchain import run_export_my_pubkey @@ -104,9 +106,11 @@ def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): lines = out.splitlines() assert config['keypair']['public'] in lines assert 'Charlie_Bucket' in lines + mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): +def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch, + mocked_setup_logging): from bigchaindb import config from bigchaindb.commands.bigchain import run_export_my_pubkey @@ -122,41 +126,49 @@ def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): # https://docs.python.org/3/library/exceptions.html#SystemExit assert exc_info.value.code == \ "This node's public key wasn't set anywhere so it can't be exported" + mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_run_init_when_db_exists(mock_db_init_with_existing_db): +def test_bigchain_run_init_when_db_exists(mocked_setup_logging, + mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_init args = Namespace(config=None) run_init(args) + mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_when_assumed_yes(mock_db_drop): +def test_drop_db_when_assumed_yes(mock_db_drop, mocked_setup_logging): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=True) run_drop(args) assert mock_db_drop.called + mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): +def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=False) monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'y') run_drop(args) assert mock_db_drop.called + mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch): +def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch, + mocked_setup_logging): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=False) monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'n') run_drop(args) assert not mock_db_drop.called + mocked_setup_logging.assert_called_once_with(user_log_config={}) def test_run_configure_when_config_exists_and_skipping(monkeypatch): @@ -245,7 +257,7 @@ def test_allow_temp_keypair_generates_one_on_the_fly( args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True) run_start(args) - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) assert bigchaindb.config['keypair']['private'] == 'private_key' assert bigchaindb.config['keypair']['public'] == 'public_key' @@ -270,7 +282,7 @@ def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True) run_start(args) - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) assert bigchaindb.config['keypair']['private'] == original_private_key assert bigchaindb.config['keypair']['public'] == original_public_key @@ -289,7 +301,7 @@ def test_run_start_when_db_already_exists(mocker, monkeypatch.setattr( 'bigchaindb.commands.bigchain._run_init', mock_run_init) run_start(run_start_args) - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) assert mocked_start.called @@ -311,7 +323,7 @@ def test_run_start_when_keypair_not_found(mocker, with pytest.raises(SystemExit) as exc: run_start(run_start_args) - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) assert len(exc.value.args) == 1 assert exc.value.args[0] == CANNOT_START_KEYPAIR_NOT_FOUND assert not mocked_start.called @@ -337,7 +349,7 @@ def test_run_start_when_start_rethinkdb_fails(mocker, with pytest.raises(SystemExit) as exc: run_start(run_start_args) - mocked_setup_logging.assert_called_once_with() + mocked_setup_logging.assert_called_once_with(user_log_config={}) assert len(exc.value.args) == 1 assert exc.value.args[0] == RETHINKDB_STARTUP_ERROR.format(err_msg) assert not mocked_start.called @@ -405,7 +417,7 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, @pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.commands.bigchain.add_replicas') -def test_run_add_replicas(mock_add_replicas): +def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): from bigchaindb.commands.bigchain import run_add_replicas from bigchaindb.backend.exceptions import OperationError @@ -415,7 +427,9 @@ def test_run_add_replicas(mock_add_replicas): mock_add_replicas.return_value = None assert run_add_replicas(args) is None assert mock_add_replicas.call_count == 1 + mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() + mocked_setup_logging.reset_mock() # test add_replicas with `OperationError` mock_add_replicas.side_effect = OperationError('err') @@ -423,7 +437,9 @@ def test_run_add_replicas(mock_add_replicas): run_add_replicas(args) assert exc.value.args == ('err',) assert mock_add_replicas.call_count == 1 + mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() + mocked_setup_logging.reset_mock() # test add_replicas with `NotImplementedError` mock_add_replicas.side_effect = NotImplementedError('err') @@ -431,12 +447,14 @@ def test_run_add_replicas(mock_add_replicas): run_add_replicas(args) assert exc.value.args == ('err',) assert mock_add_replicas.call_count == 1 + mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() + mocked_setup_logging.reset_mock() @pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.commands.bigchain.remove_replicas') -def test_run_remove_replicas(mock_remove_replicas): +def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): from bigchaindb.commands.bigchain import run_remove_replicas from bigchaindb.backend.exceptions import OperationError @@ -446,6 +464,8 @@ def test_run_remove_replicas(mock_remove_replicas): mock_remove_replicas.return_value = None assert run_remove_replicas(args) is None assert mock_remove_replicas.call_count == 1 + mocked_setup_logging.assert_called_once_with(user_log_config={}) + mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() # test add_replicas with `OperationError` @@ -454,6 +474,8 @@ def test_run_remove_replicas(mock_remove_replicas): run_remove_replicas(args) assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 + mocked_setup_logging.assert_called_once_with(user_log_config={}) + mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() # test add_replicas with `NotImplementedError` @@ -462,4 +484,6 @@ def test_run_remove_replicas(mock_remove_replicas): run_remove_replicas(args) assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 + mocked_setup_logging.assert_called_once_with(user_log_config={}) + mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index 6879e0eb..c8519d52 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -12,7 +12,7 @@ def reset_bigchaindb_config(monkeypatch): @pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config') -def test_configure_bigchaindb_configures_bigchaindb(): +def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): from bigchaindb.commands.utils import configure_bigchaindb from bigchaindb.config_utils import is_configured assert not is_configured() @@ -23,26 +23,31 @@ def test_configure_bigchaindb_configures_bigchaindb(): args = Namespace(config=None) test_configure(args) + mocked_setup_logging.assert_called_once_with(user_log_config={}) @pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config', 'reset_logging_config') @pytest.mark.parametrize('log_level', ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')) -def test_configure_bigchaindb_configures_logging(log_level): +def test_configure_bigchaindb_configures_logging(log_level, + mocked_setup_sub_logger): import logging from logging import getLogger from bigchaindb.commands.utils import configure_bigchaindb + from bigchaindb.log.configs import PUBLISHER_LOGGING_CONFIG root_logger = getLogger() assert root_logger.level == 0 @configure_bigchaindb def test_configure_logger(args): root_logger = getLogger() - assert root_logger.level == getattr(logging, log_level) + assert root_logger.level == PUBLISHER_LOGGING_CONFIG['root']['level'] args = Namespace(config=None, log_level=log_level) test_configure_logger(args) + mocked_setup_sub_logger.assert_called_once_with( + user_log_config={'level_console': log_level}) def test_start_raises_if_command_not_implemented(): diff --git a/tests/conftest.py b/tests/conftest.py index 210d526e..26beac11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -442,3 +442,15 @@ def db_name(db_config): def db_conn(): from bigchaindb.backend import connect return connect() + + +@pytest.fixture +def mocked_setup_pub_logger(mocker): + return mocker.patch( + 'bigchaindb.log.setup.setup_pub_logger', autospec=True, spec_set=True) + + +@pytest.fixture +def mocked_setup_sub_logger(mocker): + return mocker.patch( + 'bigchaindb.log.setup.setup_sub_logger', autospec=True, spec_set=True) diff --git a/tests/log/test_setup.py b/tests/log/test_setup.py index e0434eb9..39a55995 100644 --- a/tests/log/test_setup.py +++ b/tests/log/test_setup.py @@ -30,18 +30,6 @@ def mocked_socket_server(mocker): ) -@fixture -def mocked_setup_pub_logger(mocker): - return mocker.patch( - 'bigchaindb.log.setup.setup_pub_logger', autospec=True, spec_set=True) - - -@fixture -def mocked_setup_sub_logger(mocker): - return mocker.patch( - 'bigchaindb.log.setup.setup_sub_logger', autospec=True, spec_set=True) - - @fixture def log_record_dict(): return { @@ -225,6 +213,7 @@ class TestLogRecordSocketServer: assert server.server_address == ( '127.0.0.1', logging.handlers.DEFAULT_TCP_LOGGING_PORT) assert server.RequestHandlerClass == LogRecordStreamHandler + server.server_close() @mark.parametrize('side_effect', (None, KeyboardInterrupt)) def test_server_forever(self, mocker, side_effect): From 87758b8a64fefa53979aed315632d1bd11f90627 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 20 Mar 2017 15:49:24 +0100 Subject: [PATCH 28/80] Re-organize imports --- tests/commands/test_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index c8519d52..fab67cb6 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -1,7 +1,10 @@ import argparse +from argparse import ArgumentTypeError, Namespace +import logging +from logging import getLogger + import pytest -from argparse import ArgumentTypeError, Namespace from unittest.mock import patch @@ -32,8 +35,6 @@ def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): @pytest.mark.parametrize('log_level', ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')) def test_configure_bigchaindb_configures_logging(log_level, mocked_setup_sub_logger): - import logging - from logging import getLogger from bigchaindb.commands.utils import configure_bigchaindb from bigchaindb.log.configs import PUBLISHER_LOGGING_CONFIG root_logger = getLogger() From f740ebc7ce96ec49875a2f8a4efb6986d5f92f5f Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 20 Mar 2017 15:52:29 +0100 Subject: [PATCH 29/80] Use logging module constants for levels --- tests/commands/test_utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index fab67cb6..223c1f99 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -32,13 +32,19 @@ def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): @pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config', 'reset_logging_config') -@pytest.mark.parametrize('log_level', ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')) +@pytest.mark.parametrize('log_level', ( + logging.DEBUG, + logging.INFO, + logging.WARNING, + logging.ERROR, + logging.CRITICAL, +)) def test_configure_bigchaindb_configures_logging(log_level, mocked_setup_sub_logger): from bigchaindb.commands.utils import configure_bigchaindb from bigchaindb.log.configs import PUBLISHER_LOGGING_CONFIG root_logger = getLogger() - assert root_logger.level == 0 + assert root_logger.level == logging.NOTSET @configure_bigchaindb def test_configure_logger(args): From e4ed122a1c56a25109aa323b166994921f2fb82d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 14:08:38 +0100 Subject: [PATCH 30/80] Correct default log datefmt for console --- bigchaindb/log/configs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bigchaindb/log/configs.py b/bigchaindb/log/configs.py index 7a8acc7c..1be5c485 100644 --- a/bigchaindb/log/configs.py +++ b/bigchaindb/log/configs.py @@ -18,9 +18,8 @@ SUBSCRIBER_LOGGING_CONFIG = { 'formatters': { 'console': { 'class': 'logging.Formatter', - 'format': ( - '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - ), + 'format': ('[%(asctime)s] [%(levelname)s] (%(name)s) ' + '%(message)s (%(processName)-10s - pid: %(process)d)'), 'datefmt': '%Y-%m-%d %H:%M:%S', }, 'file': { From 05db44a6366385f0210b869abc8cc05fb1550836 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 14:09:06 +0100 Subject: [PATCH 31/80] Add documentation for log configuration --- .../source/server-reference/configuration.md | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index f12b8247..2c94e870 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -22,6 +22,15 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_CONFIG_PATH`
`BIGCHAINDB_BACKLOG_REASSIGN_DELAY`
`BIGCHAINDB_CONSENSUS_PLUGIN`
+`BIGCHAINDB_LOG`
+`BIGCHAINDB_LOG_FILE`
+`BIGCHAINDB_LOG_LEVEL_CONSOLE`
+`BIGCHAINDB_LOG_LEVEL_LOGFILE`
+`BIGCHAINDB_LOG_DATEFMT_CONSOLE`
+`BIGCHAINDB_LOG_DATEFMT_LOGFILE`
+`BIGCHAINDB_LOG_FMT_CONSOLE`
+`BIGCHAINDB_LOG_FMT_LOGFILE`
+`BIGCHAINDB_LOG_GRANULAR_LEVELS`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -173,3 +182,209 @@ export BIGCHAINDB_CONSENSUS_PLUGIN=default ```js "consensus_plugin": "default" ``` + +## log +The `log` key is expected to point to a mapping (set of key/value pairs) +holding the logging configuration. + +**Example**: + +``` +{ + "log": { + "file": "/var/log/bigchaindb.log", + "level_console": "info", + "level_logfile": "info", + "datefmt_console": "%Y-%m-%d %H:%M:%S", + "datefmt_logfile": "%Y-%m-%d %H:%M:%S", + "fmt_console": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "fmt_logfile": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "granular_levels": { + "bichaindb.backend": "info", + "bichaindb.core": "info" + } +} +``` + +**Defaults to**: `"{}"`. + +Please note that although the default is `"{}"` as per the configuration file, +internal defaults are used, such that the actual operational default is: + +``` +{ + "log": { + "file": "~/bigchaindb.log", + "level_console": "info", + "level_logfile": "info", + "datefmt_console": "%Y-%m-%d %H:%M:%S", + "datefmt_logfile": "%Y-%m-%d %H:%M:%S", + "fmt_console": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "fmt_logfile": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "granular_levels": {} +} +``` + +The next subsections explain each field of the `log` configuration. + + +### log.file +The full path to the file where logs should be written to. + +**Example**: + +``` +{ + "log": { + "file": "/var/log/bigchaindb/bigchaindb.log" + } +} +``` + +**Defaults to**: `"~/bigchaindb.log"`. + +Please note that the user running `bigchaindb` must have write access to the +location. + + +### log.level_console +The log level used to log to the console. Possible allowed values are the ones +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): + +``` +"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +``` + +**Example**: + +``` +{ + "log": { + "level_console": "info" + } +} +``` + +**Defaults to**: `"info"`. + + +### log.level_logfile +The log level used to log to the log file. Possible allowed values are the ones +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): + +``` +"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +``` + +**Example**: + +``` +{ + "log": { + "level_file": "info" + } +} +``` + +**Defaults to**: `"info"`. + + +### log.datefmt_console +The format string for the date/time portion of a message, when logged to the +console. + +**Example**: + +``` +{ + "log": { + "datefmt_console": "%x %X %Z" + } +} +``` + +**Defaults to**: `"%Y-%m-%d %H:%M:%S"`. + +For more information on how to construct the format string please consult the +table under Python's documentation of + [`time.strftime(format[, t])`](https://docs.python.org/3.6/library/time.html#time.strftime) + +### log.datefmt_logfile +The format string for the date/time portion of a message, when logged to a log + file. + +**Example**: + +``` +{ + "log": { + "datefmt_logfile": "%c %z" + } +} +``` + +**Defaults to**: `"%Y-%m-%d %H:%M:%S"`. + +For more information on how to construct the format string please consult the +table under Python's documentation of + [`time.strftime(format[, t])`](https://docs.python.org/3.6/library/time.html#time.strftime) + + +### log.fmt_console +A string used to format the log messages when logged to the console. + +**Example**: + +``` +{ + "log": { + "fmt_console": "%(asctime)s [%(levelname)s] %(message)s %(process)d" + } +} +``` + +**Defaults to**: `"[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)"` + +For more information on possible formatting options please consult Python's +documentation on +[LogRecord attributes](https://docs.python.org/3.6/library/logging.html#logrecord-attributes) + + +### log.fmt_logfile +A string used to format the log messages when logged to a log file. + +**Example**: + +``` +{ + "log": { + "fmt_logfile": "%(asctime)s [%(levelname)s] %(message)s %(process)d" + } +} +``` + +**Defaults to**: `"[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)"` + +For more information on possible formatting options please consult Python's +documentation on +[LogRecord attributes](https://docs.python.org/3.6/library/logging.html#logrecord-attributes) + + +### log.granular_levels +Log levels for BigchainDB's modules. This can be useful to control the log +level of specific parts of the application. As an example, if you wanted the +`core.py` to be more verbose, you would set the configuration shown in the +example below. + +**Example**: + +``` +{ + "log": { + "granular_levels": { + "bichaindb.core": "debug" + } +} +``` + +**Defaults to**: `"{}"` From d867983a891fcb2c9f4fa1881da78bc581b37ba3 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 14:37:26 +0100 Subject: [PATCH 32/80] Document cmd line option for the log level --- docs/server/source/server-reference/bigchaindb-cli.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 9612fd30..88294621 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -68,6 +68,14 @@ You can also use the `--dev-start-rethinkdb` command line option to automaticall e.g. `bigchaindb --dev-start-rethinkdb start`. Note that this will also shutdown rethinkdb when the bigchaindb process stops. The option `--dev-allow-temp-keypair` will generate a keypair on the fly if no keypair is found, this is useful when you want to run a temporary instance of BigchainDB in a Docker container, for example. +### Options +The log level for the console can be set via the option `--log-level` or its +abbreviation `-l`. Example: + +```bash +$ bigchaindb --log-level INFO start +``` + ## bigchaindb set-shards From 433863798325beefd5ab8c1c77b055b4ec5641fd Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:43:00 +0100 Subject: [PATCH 33/80] Add docs for allowed log levels for cmd line --- docs/server/source/server-reference/bigchaindb-cli.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 88294621..31c955cd 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -76,6 +76,11 @@ abbreviation `-l`. Example: $ bigchaindb --log-level INFO start ``` +The allowed levels are `DEBUG`, `INFO` , `WARNING`, `ERROR`, and `CRITICAL`. +For an explanation regarding these levels please consult the +[Logging Levels](https://docs.python.org/3.6/library/logging.html#levels) +section of Python's documentation. + ## bigchaindb set-shards From 3a812701010c8d4e2f73ccadc17eba43763bbe80 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:54:18 +0100 Subject: [PATCH 34/80] Add link to configuration file settings --- docs/server/source/server-reference/bigchaindb-cli.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 31c955cd..05f321f9 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -81,6 +81,9 @@ For an explanation regarding these levels please consult the [Logging Levels](https://docs.python.org/3.6/library/logging.html#levels) section of Python's documentation. +For a more fine-grained control over the logging configuration you can use the +configuration file as documented under +[Configuration Settings](configuration.html). ## bigchaindb set-shards From 45ae58448f05793ce34517457db5fe7ffe045406 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:54:35 +0100 Subject: [PATCH 35/80] Correct wording --- docs/server/source/server-reference/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 2c94e870..32a6c3a0 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -373,8 +373,8 @@ documentation on ### log.granular_levels Log levels for BigchainDB's modules. This can be useful to control the log level of specific parts of the application. As an example, if you wanted the -`core.py` to be more verbose, you would set the configuration shown in the -example below. +logging of the `core.py` module to be more verbose, you would set the + configuration shown in the example below. **Example**: From 9987041ac04465ed4c53957ae8746ef38097cfba Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:58:18 +0100 Subject: [PATCH 36/80] Add note about case insensitivity of log levels --- docs/server/source/server-reference/configuration.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 32a6c3a0..4cd9e9d4 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -249,10 +249,11 @@ location. ### log.level_console The log level used to log to the console. Possible allowed values are the ones -defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), +but case insensitive for convenience's sake: ``` -"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +"critical", "error", "warning", "info", "debug", "notset" ``` **Example**: @@ -270,10 +271,11 @@ defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): ### log.level_logfile The log level used to log to the log file. Possible allowed values are the ones -defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), +but case insensitive for convenience's sake: ``` -"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +"critical", "error", "warning", "info", "debug", "notset" ``` **Example**: From 10d83c2ab90822f3819eced18c701e0570e4cf84 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 22 Mar 2017 14:25:16 +0100 Subject: [PATCH 37/80] No duplicate vote inserts with mongodb (#1258) * prevent duplicate vote inserts --- bigchaindb/backend/mongodb/schema.py | 3 ++- tests/backend/mongodb/test_queries.py | 18 ++++++++++++++++++ tests/db/test_bigchain_api.py | 17 ----------------- tests/web/test_votes.py | 8 +++++++- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 4c5189ac..ad89f9bc 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -100,4 +100,5 @@ def create_votes_secondary_index(conn, dbname): ASCENDING), ('node_pubkey', ASCENDING)], - name='block_and_voter') + name='block_and_voter', + unique=True) diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index 1d7bfc39..bd7e75f1 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -212,6 +212,7 @@ def test_get_owned_ids(signed_create_tx, user_pk): def test_get_votes_by_block_id(signed_create_tx, structurally_valid_vote): + from bigchaindb.common.crypto import generate_key_pair from bigchaindb.backend import connect, query from bigchaindb.models import Block conn = connect() @@ -219,10 +220,14 @@ def test_get_votes_by_block_id(signed_create_tx, structurally_valid_vote): # create and insert a block block = Block(transactions=[signed_create_tx]) conn.db.bigchain.insert_one(block.to_dict()) + # create and insert some votes structurally_valid_vote['vote']['voting_for_block'] = block.id conn.db.votes.insert_one(structurally_valid_vote) + # create a second vote under a different key + _, pk = generate_key_pair() structurally_valid_vote['vote']['voting_for_block'] = block.id + structurally_valid_vote['node_pubkey'] = pk structurally_valid_vote.pop('_id') conn.db.votes.insert_one(structurally_valid_vote) @@ -325,6 +330,19 @@ def test_write_vote(structurally_valid_vote): assert vote_db == structurally_valid_vote +def test_duplicate_vote_raises_duplicate_key(structurally_valid_vote): + from bigchaindb.backend import connect, query + from bigchaindb.backend.exceptions import DuplicateKeyError + conn = connect() + + # write a vote + query.write_vote(conn, structurally_valid_vote) + + # write the same vote a second time + with pytest.raises(DuplicateKeyError): + query.write_vote(conn, structurally_valid_vote) + + def test_get_genesis_block(genesis_block): from bigchaindb.backend import connect, query conn = connect() diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 50d3f7b6..3f05385c 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -446,23 +446,6 @@ class TestBigchainApi(object): b.write_vote(b.vote(block_3.id, b.get_last_voted_block().id, True)) assert b.get_last_voted_block().id == block_3.id - def test_no_vote_written_if_block_already_has_vote(self, b, genesis_block): - from bigchaindb.models import Block - - block_1 = dummy_block() - b.write_block(block_1) - - b.write_vote(b.vote(block_1.id, genesis_block.id, True)) - retrieved_block_1 = b.get_block(block_1.id) - retrieved_block_1 = Block.from_dict(retrieved_block_1) - - # try to vote again on the retrieved block, should do nothing - b.write_vote(b.vote(retrieved_block_1.id, genesis_block.id, True)) - retrieved_block_2 = b.get_block(block_1.id) - retrieved_block_2 = Block.from_dict(retrieved_block_2) - - assert retrieved_block_1 == retrieved_block_2 - @pytest.mark.usefixtures('inputs') def test_assign_transaction_one_node(self, b, user_pk, user_sk): from bigchaindb.backend import query diff --git a/tests/web/test_votes.py b/tests/web/test_votes.py index bae31b9a..0bdd1081 100644 --- a/tests/web/test_votes.py +++ b/tests/web/test_votes.py @@ -27,6 +27,8 @@ def test_get_votes_endpoint(b, client): @pytest.mark.bdb @pytest.mark.usefixtures('inputs') def test_get_votes_endpoint_multiple_votes(b, client): + from bigchaindb.common.crypto import generate_key_pair + tx = Transaction.create([b.me], [([b.me], 1)]) tx = tx.sign([b.me_private]) @@ -37,8 +39,12 @@ def test_get_votes_endpoint_multiple_votes(b, client): vote_valid = b.vote(block.id, last_block, True) b.write_vote(vote_valid) - # vote the block valid + # vote the block invalid + # a note can only vote once so we need a new node_pubkey for the second + # vote + _, pk = generate_key_pair() vote_invalid = b.vote(block.id, last_block, False) + vote_invalid['node_pubkey'] = pk b.write_vote(vote_invalid) res = client.get(VOTES_ENDPOINT + '?block_id=' + block.id) From 425397f644f7f6427cd617dfc66a8386c5586ac1 Mon Sep 17 00:00:00 2001 From: Krish Date: Wed, 22 Mar 2017 14:25:25 +0100 Subject: [PATCH 38/80] NGINX frontend for MongoDB and BigchainDB (#1304) - Added NGINX deployment to frontend both BDB and MDB. - Nginx is configured with a whitelist (which is read from a ConfigMap) to allow only other MDB nodes in the closter to communicate with it. - Azure LB apparently does not support proxy protocol and hence whitelisting fails as nginx always observer the LB IP instead of the real IP in the TCP stream. - Whitelisting source IPs for MongoDB - Removing deprecated folder - Better log format - Intuitive port number usage - README and examples - Addressed a typo in PYTHON_STYLE_GUIDE.md - Azure LB apparently does not support proxy protocol and hence whitelisting fails as nginx always observer the LB IP instead of the real IP in the TCP stream. - Whitelisting source IPs for MongoDB - Removing deprecated folder - Multiple changes: - Better log format - Intuitive port number usage - README and examples - Addressed a typo in PYTHON_STYLE_GUIDE.md - Documentation - add the k8s directory to the ignore list in codecov.yml --- PYTHON_STYLE_GUIDE.md | 4 +- codecov.yml | 1 + .../add-node-on-kubernetes.rst | 15 ++ .../node-on-kubernetes.rst | 121 ++++++++++++---- .../template-kubernetes-azure.rst | 2 +- k8s/bigchaindb/bigchaindb-dep.yaml | 20 +-- k8s/deprecated.to.del/bdb-mdb-dep.yaml | 89 ------------ k8s/deprecated.to.del/bdb-rdb-dep.yaml | 87 ------------ k8s/deprecated.to.del/mongo-statefulset.yaml | 57 -------- k8s/deprecated.to.del/node-mdb-ss.yaml | 114 --------------- k8s/deprecated.to.del/node-rdb-ss.yaml | 131 ------------------ k8s/deprecated.to.del/node-ss.yaml | 89 ------------ k8s/deprecated.to.del/rethinkdb-ss.yaml | 75 ---------- k8s/mongodb/container/README.md | 2 +- k8s/mongodb/mongo-ss.yaml | 21 +-- k8s/nginx/container/Dockerfile | 11 ++ k8s/nginx/container/README.md | 70 ++++++++++ k8s/nginx/container/nginx.conf.template | 108 +++++++++++++++ k8s/nginx/container/nginx_entrypoint.bash | 44 ++++++ k8s/nginx/nginx-cm.yaml | 13 ++ k8s/nginx/nginx-dep.yaml | 82 +++++++++++ 21 files changed, 462 insertions(+), 694 deletions(-) delete mode 100644 k8s/deprecated.to.del/bdb-mdb-dep.yaml delete mode 100644 k8s/deprecated.to.del/bdb-rdb-dep.yaml delete mode 100644 k8s/deprecated.to.del/mongo-statefulset.yaml delete mode 100644 k8s/deprecated.to.del/node-mdb-ss.yaml delete mode 100644 k8s/deprecated.to.del/node-rdb-ss.yaml delete mode 100644 k8s/deprecated.to.del/node-ss.yaml delete mode 100644 k8s/deprecated.to.del/rethinkdb-ss.yaml create mode 100644 k8s/nginx/container/Dockerfile create mode 100644 k8s/nginx/container/README.md create mode 100644 k8s/nginx/container/nginx.conf.template create mode 100755 k8s/nginx/container/nginx_entrypoint.bash create mode 100644 k8s/nginx/nginx-cm.yaml create mode 100644 k8s/nginx/nginx-dep.yaml diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index befe4eeb..5ca44e83 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -82,6 +82,6 @@ flake8 --max-line-length 119 bigchaindb/ ## Writing and Running (Python) Tests -The content of this section was moved to [`bigchiandb/tests/README.md`](./tests/README.md). +The content of this section was moved to [`bigchaindb/tests/README.md`](./tests/README.md). -Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions. \ No newline at end of file +Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions. diff --git a/codecov.yml b/codecov.yml index b6f22af9..547c6b99 100644 --- a/codecov.yml +++ b/codecov.yml @@ -32,6 +32,7 @@ coverage: - "benchmarking-tests/*" - "speed-tests/*" - "ntools/*" + - "k8s/*" comment: # @stevepeak (from codecov.io) suggested we change 'suggestions' to 'uncovered' diff --git a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst index ea435ed3..7dcf1104 100644 --- a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst @@ -161,3 +161,18 @@ zero downtime during updates. You can SSH to an existing BigchainDB instance and run the ``bigchaindb show-config`` command to check that the keyring is updated. + + +Step 7: Run NGINX as a Deployment +--------------------------------- + +Please refer :ref:`this ` to +set up NGINX in your new node. + + +Step 8: Test Your New BigchainDB Node +------------------------------------- + +Please refer to the testing steps :ref:`here ` to verify that your new BigchainDB node is working as expected. + diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst index b19d79a3..6a59c750 100644 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -195,9 +195,9 @@ which can also be obtained using the ``az account list-locations`` command. You can also try to assign a name to an Public IP in Azure before starting the process, or use ``nslookup`` with the name you have in mind to check if it's available for use. -In the rare chance that name in the ``data.fqdn`` field is not available, -you must create a ConfigMap with a unique name and restart the -MongoDB instance. + +You should ensure that the the name specified in the ``data.fqdn`` field is +a unique one. **Kubernetes on bare-metal or other cloud providers.** You need to provide the name resolution function @@ -343,8 +343,8 @@ Get the file ``bigchaindb-dep.yaml`` from GitHub using: $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/bigchaindb/bigchaindb-dep.yaml -Note that we set the ``BIGCHAINDB_DATABASE_HOST`` to ``mdb`` which is the name -of the MongoDB service defined earlier. +Note that we set the ``BIGCHAINDB_DATABASE_HOST`` to ``mdb-svc`` which is the +name of the MongoDB service defined earlier. We also hardcode the ``BIGCHAINDB_KEYPAIR_PUBLIC``, ``BIGCHAINDB_KEYPAIR_PRIVATE`` and ``BIGCHAINDB_KEYRING`` for now. @@ -367,22 +367,55 @@ Create the required Deployment using: You can check its status using the command ``kubectl get deploy -w`` -Step 10: Verify the BigchainDB Node Setup +Step 10: Run NGINX as a Deployment +---------------------------------- + +NGINX is used as a proxy to both the BigchainDB and MongoDB instances in the +node. +It proxies HTTP requests on port 80 to the BigchainDB backend, and TCP +connections on port 27017 to the MongoDB backend. + +You can also configure a whitelist in NGINX to allow only connections from +other instances in the MongoDB replica set to access the backend MongoDB +instance. + +Get the file ``nginx-cm.yaml`` from GitHub using: + +.. code:: bash + + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/nginx/nginx-cm.yaml + +The IP address whitelist can be explicitly configured in ``nginx-cm.yaml`` +file. You will need a list of the IP addresses of all the other MongoDB +instances in the cluster. If the MongoDB intances specify a hostname, then this +needs to be resolved to the corresponding IP addresses. If the IP address of +any MongoDB instance changes, we can start a 'rolling upgrade' of NGINX after +updating the corresponding ConfigMap without affecting availabilty. + + +Create the ConfigMap for the whitelist using: + +.. code:: bash + + $ kubectl apply -f nginx-cm.yaml + +Get the file ``nginx-dep.yaml`` from GitHub using: + +.. code:: bash + + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/nginx/nginx-dep.yaml + +Create the NGINX deployment using: + +.. code:: bash + + $ kubectl apply -f nginx-dep.yaml + + +Step 11: Verify the BigchainDB Node Setup ----------------------------------------- -Step 10.1: Testing Externally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Try to access the ``:9984`` -on your browser. You must receive a json output that shows the BigchainDB -server version among other things. - -Try to access the ``:27017`` -on your browser. You must receive a message from MongoDB stating that it -doesn't allow HTTP connections to the port anymore. - - -Step 10.2: Testing Internally +Step 11.1: Testing Internally ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Run a container that provides utilities like ``nslookup``, ``curl`` and ``dig`` @@ -392,23 +425,53 @@ on the cluster and query the internal DNS and IP endpoints. $ kubectl run -it toolbox -- image --restart=Never --rm -It will drop you to the shell prompt. -Now you can query for the ``mdb`` and ``bdb`` service details. - -.. code:: bash - - $ nslookup mdb - $ dig +noall +answer _mdb-port._tcp.mdb.default.svc.cluster.local SRV - $ curl -X GET http://mdb:27017 - $ curl -X GET http://bdb:9984 - There is a generic image based on alpine:3.5 with the required utilities hosted at Docker Hub under ``bigchaindb/toolbox``. The corresponding Dockerfile is `here `_. + You can use it as below to get started immediately: .. code:: bash $ kubectl run -it toolbox --image bigchaindb/toolbox --restart=Never --rm +It will drop you to the shell prompt. +Now you can query for the ``mdb`` and ``bdb`` service details. + +.. code:: bash + + # nslookup mdb-svc + # nslookup bdb-svc + # nslookup ngx-svc + # dig +noall +answer _mdb-port._tcp.mdb-svc.default.svc.cluster.local SRV + # dig +noall +answer _bdb-port._tcp.bdb-svc.default.svc.cluster.local SRV + # dig +noall +answer _ngx-public-mdb-port._tcp.ngx-svc.default.svc.cluster.local SRV + # dig +noall +answer _ngx-public-bdb-port._tcp.ngx-svc.default.svc.cluster.local SRV + # curl -X GET http://mdb-svc:27017 + # curl -X GET http://bdb-svc:9984 + # curl -X GET http://ngx-svc:80 + # curl -X GET http://ngx-svc:27017 + +The ``nslookup`` commands should output the configured IP addresses of the +services in the cluster + +The ``dig`` commands should return the port numbers configured for the +various services in the cluster. + +Finally, the ``curl`` commands test the availability of the services +themselves. + +Step 11.2: Testing Externally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Try to access the ``:80`` +on your browser. You must receive a json output that shows the BigchainDB +server version among other things. + +Try to access the ``:27017`` +on your browser. If your IP is in the whitelist, you will receive a message +from the MongoDB instance stating that it doesn't allow HTTP connections to +the port anymore. If your IP is not in the whitelist, your access will be +blocked and you will not see any response from the MongoDB instance. + diff --git a/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst b/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst index 93cf1e08..b967e764 100644 --- a/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst +++ b/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst @@ -168,7 +168,7 @@ using something like: .. code:: bash - $ ssh ssh ubuntu@k8s-agent-4AC80E97-0 + $ ssh ubuntu@k8s-agent-4AC80E97-0 where ``k8s-agent-4AC80E97-0`` is the name of a Kubernetes agent node in your Kubernetes cluster. diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 7bf68f06..83daaaaf 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -1,44 +1,47 @@ ############################################################### # This config file runs bigchaindb:master as a k8s Deployment # -# and it connects to the mongodb backend on a separate pod # +# and it connects to the mongodb backend running as a # +# separate pod # ############################################################### apiVersion: v1 kind: Service metadata: - name: bdb + name: bdb-svc namespace: default labels: - name: bdb + name: bdb-svc spec: selector: - app: bdb + app: bdb-dep ports: - port: 9984 targetPort: 9984 name: bdb-port - type: LoadBalancer + type: ClusterIP + clusterIP: None --- apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: bdb + name: bdb-dep spec: replicas: 1 template: metadata: labels: - app: bdb + app: bdb-dep spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb image: bigchaindb/bigchaindb:master + imagePullPolicy: IfNotPresent args: - start env: - name: BIGCHAINDB_DATABASE_HOST - value: mdb + value: mdb-svc - name: BIGCHAINDB_DATABASE_PORT # TODO(Krish): remove hardcoded port value: "27017" @@ -58,7 +61,6 @@ spec: value: "120" - name: BIGCHAINDB_KEYRING value: "" - imagePullPolicy: IfNotPresent ports: - containerPort: 9984 hostPort: 9984 diff --git a/k8s/deprecated.to.del/bdb-mdb-dep.yaml b/k8s/deprecated.to.del/bdb-mdb-dep.yaml deleted file mode 100644 index c985b285..00000000 --- a/k8s/deprecated.to.del/bdb-mdb-dep.yaml +++ /dev/null @@ -1,89 +0,0 @@ -############################################################### -# This config file runs bigchaindb:latest and connects to the # -# mongodb backend as a service # -############################################################### - -apiVersion: v1 -kind: Service -metadata: - name: bdb-mdb-service - namespace: default - labels: - name: bdb-mdb-service -spec: - selector: - app: bdb-mdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-api - type: LoadBalancer ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: bdb-mdb -spec: - replicas: 1 - template: - metadata: - labels: - app: bdb-mdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb-mdb - image: bigchaindb/bigchaindb:latest - args: - - start - env: - - name: BIGCHAINDB_DATABASE_HOST - value: mdb-service - - name: BIGCHAINDB_DATABASE_PORT - value: "27017" - - name: BIGCHAINDB_DATABASE_REPLICASET - value: bigchain-rs - - name: BIGCHIANDB_DATABASE_BACKEND - value: mongodb - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_SERVER_BIND - value: 0.0.0.0:9984 - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - - name: BIGCHAINDB_KEYRING - value: "" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - volumeMounts: - - name: bigchaindb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: bigchaindb-data - hostPath: - path: /disk/bigchaindb-data diff --git a/k8s/deprecated.to.del/bdb-rdb-dep.yaml b/k8s/deprecated.to.del/bdb-rdb-dep.yaml deleted file mode 100644 index 06daca43..00000000 --- a/k8s/deprecated.to.del/bdb-rdb-dep.yaml +++ /dev/null @@ -1,87 +0,0 @@ -############################################################### -# This config file runs bigchaindb:latest and connects to the # -# rethinkdb backend as a service # -############################################################### - -apiVersion: v1 -kind: Service -metadata: - name: bdb-rdb-service - namespace: default - labels: - name: bdb-rdb-service -spec: - selector: - app: bdb-rdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-rdb-api - type: LoadBalancer ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: bdb-rdb -spec: - replicas: 1 - template: - metadata: - labels: - app: bdb-rdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb-rdb - image: bigchaindb/bigchaindb:latest - args: - - start - env: - - name: BIGCHAINDB_DATABASE_HOST - value: rdb-service - - name: BIGCHAINDB_DATABASE_PORT - value: "28015" - - name: BIGCHIANDB_DATABASE_BACKEND - value: rethinkdb - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_SERVER_BIND - value: 0.0.0.0:9984 - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - - name: BIGCHAINDB_KEYRING - value: "" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - volumeMounts: - - name: bigchaindb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: bigchaindb-data - hostPath: - path: /disk/bigchaindb-data diff --git a/k8s/deprecated.to.del/mongo-statefulset.yaml b/k8s/deprecated.to.del/mongo-statefulset.yaml deleted file mode 100644 index a71567f3..00000000 --- a/k8s/deprecated.to.del/mongo-statefulset.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mongodb - labels: - name: mongodb -spec: - ports: - - port: 27017 - targetPort: 27017 - clusterIP: None - selector: - role: mongodb ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: mongodb -spec: - serviceName: mongodb - replicas: 3 - template: - metadata: - labels: - role: mongodb - environment: staging - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: mongo - image: mongo:3.4.1 - command: - - mongod - - "--replSet" - - bigchain-rs - #- "--smallfiles" - #- "--noprealloc" - ports: - - containerPort: 27017 - volumeMounts: - - name: mongo-persistent-storage - mountPath: /data/db - - name: mongo-sidecar - image: cvallance/mongo-k8s-sidecar - env: - - name: MONGO_SIDECAR_POD_LABELS - value: "role=mongo,environment=staging" - volumeClaimTemplates: - - metadata: - name: mongo-persistent-storage - annotations: - volume.beta.kubernetes.io/storage-class: "fast" - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 100Gi diff --git a/k8s/deprecated.to.del/node-mdb-ss.yaml b/k8s/deprecated.to.del/node-mdb-ss.yaml deleted file mode 100644 index 3c126d2d..00000000 --- a/k8s/deprecated.to.del/node-mdb-ss.yaml +++ /dev/null @@ -1,114 +0,0 @@ -################################################################# -# This YAML file desribes a StatefulSet with two containers: # -# bigchaindb/bigchaindb:latest and mongo:3.4.1 # -# It also describes a Service to expose BigchainDB and MongoDB. # -################################################################# - -apiVersion: v1 -kind: Service -metadata: - name: bdb-service - namespace: default - labels: - name: bdb-service -spec: - selector: - app: bdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-http-api - - port: 27017 - targetPort: 27017 - name: mongodb-port - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: bdb - namespace: default -spec: - serviceName: bdb - replicas: 1 - template: - metadata: - name: bdb - labels: - app: bdb - #annotations: - #pod.beta.kubernetes.io/init-containers: '[ - # TODO mongodb user and group; id = 999 - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bigchaindb - image: bigchaindb/bigchaindb:master - args: - - start - env: - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ - - name: BIGCHAINDB_KEYRING - value: "" - - name: BIGCHAINDB_DATABASE_BACKEND - value: mongodb - - name: BIGCHAINDB_DATABASE_HOST - value: localhost - - name: BIGCHAINDB_DATABASE_PORT - value: "27017" - - name: BIGCHAINDB_SERVER_BIND - value: "0.0.0.0:9984" - - name: BIGCHAINDB_DATABASE_REPLICASET - value: bigchain-rs - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: bdb-port - initialDelaySeconds: 15 - timeoutSeconds: 10 - - name: mongodb - image: mongo:3.4.1 - args: - - --replSet=bigchain-rs - imagePullPolicy: IfNotPresent - ports: - - containerPort: 27017 - hostPort: 27017 - name: mdb-port - protocol: TCP - volumeMounts: - - name: mdb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - tcpSocket: - port: mdb-port - successThreshold: 1 - failureThreshold: 3 - periodSeconds: 15 - timeoutSeconds: 1 - restartPolicy: Always - volumes: - - name: mdb-data - persistentVolumeClaim: - claimName: mongoclaim diff --git a/k8s/deprecated.to.del/node-rdb-ss.yaml b/k8s/deprecated.to.del/node-rdb-ss.yaml deleted file mode 100644 index fc157746..00000000 --- a/k8s/deprecated.to.del/node-rdb-ss.yaml +++ /dev/null @@ -1,131 +0,0 @@ -############################################################## -# This YAML file desribes a StatefulSet with two containers: # -# bigchaindb/bigchaindb:latest and rethinkdb:2.3 # -# It also describes a Service to expose BigchainDB, # -# the RethinkDB intracluster communications port, and # -# the RethinkDB web interface port. # -############################################################## - -apiVersion: v1 -kind: Service -metadata: - name: bdb-service - namespace: default - labels: - name: bdb-service -spec: - selector: - app: bdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-http-api - - port: 29015 - targetPort: 29015 - name: rdb-intracluster-comm-port - - port: 8080 - targetPort: 8080 - name: rdb-web-interface-port - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: bdb - namespace: default -spec: - serviceName: bdb - replicas: 1 - template: - metadata: - name: bdb - labels: - app: bdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb-server - image: bigchaindb/bigchaindb:latest - args: - - start - env: - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 56mEvwwVxcYsFQ3Y8UTFB8DVBv38yoUhxzDW3DAdLVd2 - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: 9DsHwiEtvk51UHmNM2eV66czFha69j3CdtNrCj1RcZWR - - name: BIGCHAINDB_KEYRING - value: "" - - name: BIGCHAINDB_DATABASE_BACKEND - value: rethinkdb - - name: BIGCHAINDB_DATABASE_HOST - value: localhost - - name: BIGCHAINDB_DATABASE_PORT - value: "28015" - - name: BIGCHAINDB_SERVER_BIND - value: "0.0.0.0:9984" - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - - name: rethinkdb - image: rethinkdb:2.3 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - hostPort: 8080 - name: rdb-web-interface-port - protocol: TCP - - containerPort: 29015 - hostPort: 29015 - name: rdb-intra-port - protocol: TCP - - containerPort: 28015 - hostPort: 28015 - name: rdb-client-port - protocol: TCP - volumeMounts: - - name: rdb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: rdb-data - persistentVolumeClaim: - claimName: mongoclaim diff --git a/k8s/deprecated.to.del/node-ss.yaml b/k8s/deprecated.to.del/node-ss.yaml deleted file mode 100644 index 9580daf6..00000000 --- a/k8s/deprecated.to.del/node-ss.yaml +++ /dev/null @@ -1,89 +0,0 @@ -##################################################### -# This config file uses bdb v0.9.1 with bundled rdb # -##################################################### - -apiVersion: v1 -kind: Service -metadata: - name: bdb-service - namespace: default - labels: - name: bdb-service -spec: - selector: - app: bdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-http-api - - port: 8080 - targetPort: 8080 - name: bdb-rethinkdb-api - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: bdb - namespace: default -spec: - serviceName: bdb - replicas: 1 - template: - metadata: - name: bdb - labels: - app: bdb - annotations: - pod.beta.kubernetes.io/init-containers: '[ - { - "name": "bdb091-configure", - "image": "bigchaindb/bigchaindb:0.9.1", - "command": ["bigchaindb", "-y", "configure", "rethinkdb"], - "volumeMounts": [ - { - "name": "bigchaindb-data", - "mountPath": "/data" - } - ] - } - ]' - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb091-server - image: bigchaindb/bigchaindb:0.9.1 - args: - - -c - - /data/.bigchaindb - - start - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - volumeMounts: - - name: bigchaindb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: bigchaindb-data - hostPath: - path: /disk/bigchaindb-data diff --git a/k8s/deprecated.to.del/rethinkdb-ss.yaml b/k8s/deprecated.to.del/rethinkdb-ss.yaml deleted file mode 100644 index 081a5f6c..00000000 --- a/k8s/deprecated.to.del/rethinkdb-ss.yaml +++ /dev/null @@ -1,75 +0,0 @@ -#################################################### -# This config file runs rethinkdb:2.3 as a service # -#################################################### - -apiVersion: v1 -kind: Service -metadata: - name: rdb-service - namespace: default - labels: - name: rdb-service -spec: - selector: - app: rdb - ports: - - port: 8080 - targetPort: 8080 - name: rethinkdb-http-port - - port: 28015 - targetPort: 28015 - name: rethinkdb-driver-port - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: rdb - namespace: default -spec: - serviceName: rdb - replicas: 1 - template: - metadata: - name: rdb - labels: - app: rdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: rethinkdb - image: rethinkdb:2.3 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - hostPort: 8080 - name: rdb-http-port - protocol: TCP - - containerPort: 28015 - hostPort: 28015 - name: rdb-client-port - protocol: TCP - volumeMounts: - - name: rdb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: rdb-data - hostPath: - path: /disk/rdb-data diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md index 7896a912..baad9f13 100644 --- a/k8s/mongodb/container/README.md +++ b/k8s/mongodb/container/README.md @@ -19,7 +19,7 @@ ``` docker run \ --name=mdb1 \ ---publish=17017:17017 \ +--publish=: \ --rm=true \ bigchaindb/mongodb \ --replica-set-name \ diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index fb6a73f8..089a0a96 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -1,38 +1,39 @@ ######################################################################## # This YAML file desribes a StatefulSet with a service for running and # -# exposing a MongoDB service. # +# exposing a MongoDB instance. # # It depends on the configdb and db k8s pvc. # ######################################################################## apiVersion: v1 kind: Service metadata: - name: mdb + name: mdb-svc namespace: default labels: - name: mdb + name: mdb-svc spec: selector: - app: mdb + app: mdb-ss ports: - port: 27017 targetPort: 27017 name: mdb-port - type: LoadBalancer + type: ClusterIP + clusterIP: None --- apiVersion: apps/v1beta1 kind: StatefulSet metadata: - name: mdb + name: mdb-ss namespace: default spec: - serviceName: mdb + serviceName: mdb-svc replicas: 1 template: metadata: - name: mdb + name: mdb-ss labels: - app: mdb + app: mdb-ss spec: terminationGracePeriodSeconds: 10 containers: @@ -41,6 +42,7 @@ spec: # versions during updates and rollbacks. Also, once fixed, change the # imagePullPolicy to IfNotPresent for faster bootup image: bigchaindb/mongodb:latest + imagePullPolicy: Always env: - name: MONGODB_FQDN valueFrom: @@ -60,7 +62,6 @@ spec: capabilities: add: - FOWNER - imagePullPolicy: Always ports: - containerPort: 27017 hostPort: 27017 diff --git a/k8s/nginx/container/Dockerfile b/k8s/nginx/container/Dockerfile new file mode 100644 index 00000000..c6c4dd3f --- /dev/null +++ b/k8s/nginx/container/Dockerfile @@ -0,0 +1,11 @@ +FROM nginx:1.11.10 +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /etc/nginx/nginx.conf +COPY nginx_entrypoint.bash / +EXPOSE 80 443 27017 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx/container/README.md b/k8s/nginx/container/README.md new file mode 100644 index 00000000..9cb44246 --- /dev/null +++ b/k8s/nginx/container/README.md @@ -0,0 +1,70 @@ +## Custom Nginx container for a Node + +### Need + +* Since, BigchainDB and MongoDB both need to expose ports to the outside + world (inter and intra cluster), we need to have a basic DDoS mitigation + strategy to ensure that we can provide proper uptime and security these + core services. + +* We can have a proxy like nginx/haproxy in every node that listens to + global connections and applies cluster level entry policy. + +### Implementation +* For MongoDB cluster communication, we will use nginx with an environment + variable specifying a ":" separated list of IPs in the whitelist. This list + contains the IPs of exising instances in the MongoDB replica set so as to + allow connections from the whitelist and avoid a DDoS. + +* For BigchainDB connections, nginx needs to have rules to throttle + connections that are using resources over a threshold. + + +### Step 1: Build the Latest Container + +Run `docker build -t bigchaindb/nginx .` from this folder. + +Optional: Upload container to Docker Hub: +`docker push bigchaindb/nginx:` + +### Step 2: Run the Container + +Note that the whilelist IPs must be specified with the subnet in the CIDR +format, eg: `1.2.3.4/16` + +``` +docker run \ +--env "MONGODB_FRONTEND_PORT=" \ +--env "MONGODB_BACKEND_HOST=" \ +--env "MONGODB_BACKEND_PORT=" \ +--env "BIGCHAINDB_FRONTEND_PORT=" \ +--env "BIGCHAINDB_BACKEND_HOST=" \ +--env "BIGCHAINDB_BACKEND_PORT=" \ +--env "MONGODB_WHITELIST=" \ +--name=ngx \ +--publish=: \ +--publish=: \ +--rm=true \ +bigchaindb/nginx +``` + +For example: +``` +docker run \ +--env "MONGODB_FRONTEND_PORT=17017" \ +--env "MONGODB_BACKEND_HOST=localhost" \ +--env "MONGODB_BACKEND_PORT=27017" \ +--env "BIGCHAINDB_FRONTEND_PORT=80" \ +--env "BIGCHAINDB_BACKEND_HOST=localhost" \ +--env "BIGCHAINDB_BACKEND_PORT=9984" \ +--env "MONGODB_WHITELIST="192.168.0.0/16:10.0.2.0/24" \ +--name=ngx \ +--publish=80:80 \ +--publish=17017:17017 \ +--rm=true \ +bigchaindb/nginx +``` + diff --git a/k8s/nginx/container/nginx.conf.template b/k8s/nginx/container/nginx.conf.template new file mode 100644 index 00000000..eda3e7c7 --- /dev/null +++ b/k8s/nginx/container/nginx.conf.template @@ -0,0 +1,108 @@ +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /etc/nginx/nginx.error.log; + +events { + worker_connections 256; + accept_mutex on; + use epoll; +} + +http { + server_names_hash_bucket_size 128; + resolver 8.8.8.8 8.8.4.4; + access_log /etc/nginx/nginx.access.log combined buffer=16k flush=5s; + + # allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # enable logging when requests are being throttled + limit_req_log_level notice; + + # the http status code to return to the client when throttling; + # 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + upstream bdb_backend { + server BIGCHAINDB_BACKEND_HOST:BIGCHAINDB_BACKEND_PORT max_fails=5 fail_timeout=30; + } + + server { + listen BIGCHAINDB_FRONTEND_PORT; + # server_name "FRONTEND_DNS_NAME"; + underscores_in_headers on; + + # max client request body size: avg transaction size + client_max_body_size 15k; + + # keepalive connection settings + keepalive_timeout 20s; + + # `slowloris` attack mitigation settings + client_body_timeout 10s; + client_header_timeout 10s; + + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_redirect off; + + # TODO proxy_set_header X-Forwarded-Proto https; + + # limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + proxy_pass http://bdb_backend; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /etc/nginx/50x.html; + } + } +} + +# NGINX stream block for TCP and UDP proxies +stream { + log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /etc/nginx/nginx.stream.access.log mdb_log buffer=16k flush=5s; + + # define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address + limit_conn_zone $binary_remote_addr zone=two:10m; + + # enable logging when connections are being throttled + limit_conn_log_level notice; + + upstream mdb_backend { + server MONGODB_BACKEND_HOST:MONGODB_BACKEND_PORT max_fails=5 fail_timeout=30 max_conns=1024; + } + + server { + listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5; + preread_timeout 30s; + tcp_nodelay on; + + # whitelist + MONGODB_WHITELIST + # deny access to everyone else + deny all; + + # allow 512 connections from the same IP address + limit_conn two 512; + + proxy_pass mdb_backend; + } +} diff --git a/k8s/nginx/container/nginx_entrypoint.bash b/k8s/nginx/container/nginx_entrypoint.bash new file mode 100755 index 00000000..9b63e278 --- /dev/null +++ b/k8s/nginx/container/nginx_entrypoint.bash @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail + +mongo_frontend_port=`printenv MONGODB_FRONTEND_PORT` +mongo_backend_host=`printenv MONGODB_BACKEND_HOST` +mongo_backend_port=`printenv MONGODB_BACKEND_PORT` +bdb_frontend_port=`printenv BIGCHAINDB_FRONTEND_PORT` +bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` +bdb_backend_port=`printenv BIGCHAINDB_BACKEND_PORT` +mongo_whitelist=`printenv MONGODB_WHITELIST` + +# sanity checks +if [[ -z "${mongo_frontend_port}" || \ + -z "${mongo_backend_host}" || \ + -z "${mongo_backend_port}" || \ + -z "${bdb_frontend_port}" || \ + -z "${bdb_backend_host}" || \ + -z "${bdb_backend_port}" ]] ; then + echo "Invalid environment settings detected. Exiting!" + exit 1 +fi + +NGINX_CONF_FILE=/etc/nginx/nginx.conf + +# configure the nginx.conf file with env variables +sed -i "s|MONGODB_FRONTEND_PORT|${mongo_frontend_port}|g" $NGINX_CONF_FILE +sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" $NGINX_CONF_FILE +sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" $NGINX_CONF_FILE +sed -i "s|BIGCHAINDB_FRONTEND_PORT|${bdb_frontend_port}|g" $NGINX_CONF_FILE +sed -i "s|BIGCHAINDB_BACKEND_HOST|${bdb_backend_host}|g" $NGINX_CONF_FILE +sed -i "s|BIGCHAINDB_BACKEND_PORT|${bdb_backend_port}|g" $NGINX_CONF_FILE + +# populate the whitelist in the conf file as per MONGODB_WHITELIST env var +hosts=$(echo ${mongo_whitelist} | tr ":" "\n") +for host in $hosts; do + sed -i "s|MONGODB_WHITELIST|allow ${host};\n MONGODB_WHITELIST|g" $NGINX_CONF_FILE +done + +# remove the MONGODB_WHITELIST marker string from template +sed -i "s|MONGODB_WHITELIST||g" $NGINX_CONF_FILE + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c /etc/nginx/nginx.conf diff --git a/k8s/nginx/nginx-cm.yaml b/k8s/nginx/nginx-cm.yaml new file mode 100644 index 00000000..7a255aae --- /dev/null +++ b/k8s/nginx/nginx-cm.yaml @@ -0,0 +1,13 @@ +######################################################################### +# This YAML file desribes a ConfigMap with a valid list of IP addresses # +# that can connect to the MongoDB instance. # +######################################################################### + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-whitelist + namespace: default +data: + # ':' separated list of allowed hosts + allowed-hosts: 192.168.0.0/16:10.0.2.0/24 diff --git a/k8s/nginx/nginx-dep.yaml b/k8s/nginx/nginx-dep.yaml new file mode 100644 index 00000000..d7739a56 --- /dev/null +++ b/k8s/nginx/nginx-dep.yaml @@ -0,0 +1,82 @@ +############################################################### +# This config file runs nginx as a k8s deployment and exposes # +# it using an external load balancer. # +# This deployment is used as a front end to both BigchainDB # +# and MongoDB. # +############################################################### + +apiVersion: v1 +kind: Service +metadata: + name: ngx-svc + namespace: default + labels: + name: ngx-svc + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-dep + ports: + - port: 27017 + targetPort: 27017 + name: ngx-public-mdb-port + protocol: TCP + - port: 80 + targetPort: 80 + name: ngx-public-bdb-port + protocol: TCP + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: ngx-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: bigchaindb/nginx:latest + imagePullPolicy: Always + env: + - name: MONGODB_FRONTEND_PORT + value: "27017" + - name: MONGODB_BACKEND_HOST + value: mdb-svc + - name: MONGODB_BACKEND_PORT + value: "27017" + - name: BIGCHAINDB_FRONTEND_PORT + value: "80" + - name: BIGCHAINDB_BACKEND_HOST + value: bdb-svc + - name: BIGCHAINDB_BACKEND_PORT + value: "9984" + - name: MONGODB_WHITELIST + valueFrom: + configMapKeyRef: + name: mongodb-whitelist + key: allowed-hosts + ports: + - containerPort: 27017 + hostPort: 27017 + name: public-mdb-port + protocol: TCP + - containerPort: 80 + hostPort: 80 + name: public-bdb-port + protocol: TCP + resources: + limits: + cpu: 200m + memory: 768Mi + #livenessProbe: TODO(Krish) + #readinessProbe: TODO(Krish) + restartPolicy: Always From f98a634d65e3be5a0aa5f06e2fea13843fc73c70 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 22 Mar 2017 14:37:37 +0100 Subject: [PATCH 39/80] clarify allowed maximum complexity of conditions --- docs/server/source/data-models/inputs-outputs.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index 9f1b5d56..e81aa3b2 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -22,7 +22,12 @@ One can also put different weights on the inputs to a threshold condition, along The (single) output of a threshold condition can be used as one of the inputs of other threshold conditions. This means that one can combine threshold conditions to build complex logical expressions, e.g. (x OR y) AND (u OR v). -When one creates a condition, one can calculate its fulfillment length (e.g. 96). The more complex the condition, the larger its fulfillment length will be. A BigchainDB federation can put an upper limit on the allowed fulfillment length, as a way of capping the complexity of conditions (and the computing time required to validate them). +When one creates a condition, one can calculate its fulfillment length (e.g. +96). The more complex the condition, the larger its fulfillment length will be. +A BigchainDB federation can put an upper limit on the complexity of the +conditions, either directly by setting an allowed maximum fulfillment length, +or indirectly by setting a maximum allowed transaction size which would limit +the overall complexity accross all inputs and outputs of a transaction. If someone tries to make a condition where the output of a threshold condition feeds into the input of another “earlier” threshold condition (i.e. in a closed logical circuit), then their computer will take forever to calculate the (infinite) “condition URI”, at least in theory. In practice, their computer will run out of memory or their client software will timeout after a while. From 0ae9d19a542a6c9882eac864ad827cf903e4fbf8 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 14:33:25 +0100 Subject: [PATCH 40/80] Separate log configuration from logging process Closes #1317 --- bigchaindb/commands/bigchain.py | 4 +- bigchaindb/commands/utils.py | 50 +++++++++++++++++++---- tests/commands/rethinkdb/test_commands.py | 16 ++------ tests/commands/test_commands.py | 42 ++++--------------- tests/commands/test_utils.py | 19 ++++----- 5 files changed, 66 insertions(+), 65 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 767f6ccc..be17d75f 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -24,7 +24,8 @@ from bigchaindb.commands.messages import ( CANNOT_START_KEYPAIR_NOT_FOUND, RETHINKDB_STARTUP_ERROR, ) -from bigchaindb.commands.utils import configure_bigchaindb, input_on_stderr +from bigchaindb.commands.utils import ( + configure_bigchaindb, start_logging_process, input_on_stderr) logging.basicConfig(level=logging.INFO) @@ -169,6 +170,7 @@ def run_drop(args): @configure_bigchaindb +@start_logging_process def run_start(args): """Start the processes to run the node""" logger.info('BigchainDB Version %s', bigchaindb.__version__) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 73313f05..f4a311fa 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -21,20 +21,56 @@ from bigchaindb.version import __version__ def configure_bigchaindb(command): + """Decorator to be used by command line functions, such that the + configuration of bigchaindb is performed before the execution of + the command. + + Args: + command: The command to decorate. + + Returns: + The command wrapper function. + + """ @functools.wraps(command) def configure(args): - bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) - - logging_config = bigchaindb.config['log'] or {} - if 'log_level' in args and args.log_level: - logging_config['level_console'] = args.log_level - setup_logging(user_log_config=logging_config) - + try: + config_from_cmdline = {'log': {'level_console': args.log_level}} + except AttributeError: + config_from_cmdline = None + bigchaindb.config_utils.autoconfigure( + filename=args.config, config=config_from_cmdline, force=True) command(args) return configure +def start_logging_process(command): + """Decorator to start the logging subscriber process. + + Args: + command: The command to decorate. + + Returns: + The command wrapper function. + + .. important:: + + Configuration, if needed, should be applied before invoking this + decorator, as starting the subscriber process for logging will + configure the root logger for the child process based on the + state of :obj:`bigchaindb.config` at the moment this decorator + is invoked. + + """ + @functools.wraps(command) + def start_logging(args): + from bigchaindb import config + setup_logging(user_log_config=config.get('log')) + command(args) + return start_logging + + # We need this because `input` always prints on stdout, while it should print # to stderr. It's a very old bug, check it out here: # - https://bugs.python.org/issue1927 diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index ac100075..165fef0d 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -38,7 +38,7 @@ def test_start_rethinkdb_exits_when_cannot_start(mock_popen): @patch('rethinkdb.ast.Table.reconfigure') -def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): +def test_set_shards(mock_reconfigure, monkeypatch, b): from bigchaindb.commands.bigchain import run_set_shards # this will mock the call to retrieve the database config @@ -50,8 +50,6 @@ def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): args = Namespace(num_shards=3, config=None) run_set_shards(args) mock_reconfigure.assert_called_with(replicas=1, shards=3, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() # this will mock the call to retrieve the database config # we will set it to return three replica @@ -61,10 +59,9 @@ def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_replicas) run_set_shards(args) mock_reconfigure.assert_called_with(replicas=3, shards=3, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_set_shards_raises_exception(monkeypatch, b, mocked_setup_logging): +def test_set_shards_raises_exception(monkeypatch, b): from bigchaindb.commands.bigchain import run_set_shards # test that we are correctly catching the exception @@ -81,11 +78,10 @@ def test_set_shards_raises_exception(monkeypatch, b, mocked_setup_logging): with pytest.raises(SystemExit) as exc: run_set_shards(args) assert exc.value.args == ('Failed to reconfigure tables.',) - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('rethinkdb.ast.Table.reconfigure') -def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): +def test_set_replicas(mock_reconfigure, monkeypatch, b): from bigchaindb.commands.bigchain import run_set_replicas # this will mock the call to retrieve the database config @@ -97,8 +93,6 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): args = Namespace(num_replicas=2, config=None) run_set_replicas(args) mock_reconfigure.assert_called_with(replicas=2, shards=2, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() # this will mock the call to retrieve the database config # we will set it to return three shards @@ -108,10 +102,9 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_shards) run_set_replicas(args) mock_reconfigure.assert_called_with(replicas=2, shards=3, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_set_replicas_raises_exception(monkeypatch, b, mocked_setup_logging): +def test_set_replicas_raises_exception(monkeypatch, b): from bigchaindb.commands.bigchain import run_set_replicas # test that we are correctly catching the exception @@ -128,4 +121,3 @@ def test_set_replicas_raises_exception(monkeypatch, b, mocked_setup_logging): with pytest.raises(SystemExit) as exc: run_set_replicas(args) assert exc.value.args == ('Failed to reconfigure tables.',) - mocked_setup_logging.assert_called_once_with(user_log_config={}) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index eebd86ea..50b995b0 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -74,7 +74,7 @@ def test_bigchain_run_start_assume_yes_create_default_config( # interfere with capsys. # See related issue: https://github.com/pytest-dev/pytest/issues/128 @pytest.mark.usefixtures('ignore_local_config_file') -def test_bigchain_show_config(capsys, mocked_setup_logging): +def test_bigchain_show_config(capsys): from bigchaindb import config from bigchaindb.commands.bigchain import run_show_config @@ -85,11 +85,9 @@ def test_bigchain_show_config(capsys, mocked_setup_logging): del config['CONFIGURED'] config['keypair']['private'] = 'x' * 45 assert output_config == config - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch, - mocked_setup_logging): +def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): from bigchaindb import config from bigchaindb.commands.bigchain import run_export_my_pubkey @@ -106,11 +104,9 @@ def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch, lines = out.splitlines() assert config['keypair']['public'] in lines assert 'Charlie_Bucket' in lines - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch, - mocked_setup_logging): +def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): from bigchaindb import config from bigchaindb.commands.bigchain import run_export_my_pubkey @@ -126,49 +122,41 @@ def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch, # https://docs.python.org/3/library/exceptions.html#SystemExit assert exc_info.value.code == \ "This node's public key wasn't set anywhere so it can't be exported" - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_run_init_when_db_exists(mocked_setup_logging, - mock_db_init_with_existing_db): +def test_bigchain_run_init_when_db_exists(mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_init args = Namespace(config=None) run_init(args) - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_when_assumed_yes(mock_db_drop, mocked_setup_logging): +def test_drop_db_when_assumed_yes(mock_db_drop): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=True) run_drop(args) assert mock_db_drop.called - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch, - mocked_setup_logging): +def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=False) monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'y') run_drop(args) assert mock_db_drop.called - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch, - mocked_setup_logging): +def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=False) monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'n') run_drop(args) assert not mock_db_drop.called - mocked_setup_logging.assert_called_once_with(user_log_config={}) def test_run_configure_when_config_exists_and_skipping(monkeypatch): @@ -417,7 +405,7 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, @pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.commands.bigchain.add_replicas') -def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): +def test_run_add_replicas(mock_add_replicas): from bigchaindb.commands.bigchain import run_add_replicas from bigchaindb.backend.exceptions import OperationError @@ -427,9 +415,7 @@ def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): mock_add_replicas.return_value = None assert run_add_replicas(args) is None assert mock_add_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() - mocked_setup_logging.reset_mock() # test add_replicas with `OperationError` mock_add_replicas.side_effect = OperationError('err') @@ -437,9 +423,7 @@ def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): run_add_replicas(args) assert exc.value.args == ('err',) assert mock_add_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() - mocked_setup_logging.reset_mock() # test add_replicas with `NotImplementedError` mock_add_replicas.side_effect = NotImplementedError('err') @@ -447,14 +431,12 @@ def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): run_add_replicas(args) assert exc.value.args == ('err',) assert mock_add_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() - mocked_setup_logging.reset_mock() @pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.commands.bigchain.remove_replicas') -def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): +def test_run_remove_replicas(mock_remove_replicas): from bigchaindb.commands.bigchain import run_remove_replicas from bigchaindb.backend.exceptions import OperationError @@ -464,8 +446,6 @@ def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): mock_remove_replicas.return_value = None assert run_remove_replicas(args) is None assert mock_remove_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() # test add_replicas with `OperationError` @@ -474,8 +454,6 @@ def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): run_remove_replicas(args) assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() # test add_replicas with `NotImplementedError` @@ -484,6 +462,4 @@ def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): run_remove_replicas(args) assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index 223c1f99..5f190717 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -1,7 +1,6 @@ import argparse from argparse import ArgumentTypeError, Namespace import logging -from logging import getLogger import pytest @@ -15,7 +14,7 @@ def reset_bigchaindb_config(monkeypatch): @pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config') -def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): +def test_configure_bigchaindb_configures_bigchaindb(): from bigchaindb.commands.utils import configure_bigchaindb from bigchaindb.config_utils import is_configured assert not is_configured() @@ -26,7 +25,6 @@ def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): args = Namespace(config=None) test_configure(args) - mocked_setup_logging.assert_called_once_with(user_log_config={}) @pytest.mark.usefixtures('ignore_local_config_file', @@ -39,22 +37,19 @@ def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): logging.ERROR, logging.CRITICAL, )) -def test_configure_bigchaindb_configures_logging(log_level, - mocked_setup_sub_logger): +def test_configure_bigchaindb_logging(log_level): from bigchaindb.commands.utils import configure_bigchaindb - from bigchaindb.log.configs import PUBLISHER_LOGGING_CONFIG - root_logger = getLogger() - assert root_logger.level == logging.NOTSET + from bigchaindb import config + assert not config['log'] @configure_bigchaindb def test_configure_logger(args): - root_logger = getLogger() - assert root_logger.level == PUBLISHER_LOGGING_CONFIG['root']['level'] + pass args = Namespace(config=None, log_level=log_level) test_configure_logger(args) - mocked_setup_sub_logger.assert_called_once_with( - user_log_config={'level_console': log_level}) + from bigchaindb import config + assert config['log'] == {'level_console': log_level} def test_start_raises_if_command_not_implemented(): From b42264e27e934d34e0bbc63bf988ee1cb5120629 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 14:46:21 +0100 Subject: [PATCH 41/80] Add commands subpackage documentation --- docs/server/source/appendices/commands.rst | 18 ++++++++++++++++++ docs/server/source/appendices/index.rst | 1 + 2 files changed, 19 insertions(+) create mode 100644 docs/server/source/appendices/commands.rst diff --git a/docs/server/source/appendices/commands.rst b/docs/server/source/appendices/commands.rst new file mode 100644 index 00000000..35d37b27 --- /dev/null +++ b/docs/server/source/appendices/commands.rst @@ -0,0 +1,18 @@ +###################### +Command Line Interface +###################### + +.. automodule:: bigchaindb.commands + :special-members: __init__ + + +:mod:`bigchaindb.commands.bigchain` +----------------------------------- + +.. automodule:: bigchaindb.commands.bigchain + + +:mod:`bigchaindb.commands.utils` +-------------------------------- + +.. automodule:: bigchaindb.commands.utils diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 365bedfa..7beb27f5 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -16,6 +16,7 @@ Appendices consensus pipelines backend + commands aws-setup generate-key-pair-for-ssh firewall-notes From 054fb48ca8f4be1b88e864cd979b19edc58f75a4 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 16:36:10 +0100 Subject: [PATCH 42/80] Set cmd line log level option default --- bigchaindb/commands/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index f4a311fa..b67c8ee8 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -196,6 +196,7 @@ base_parser.add_argument('-c', '--config', base_parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Log level') base_parser.add_argument('-y', '--yes', '--yes-please', From 6e3f25a1432eaa1ed9cc7a7e30a115130d5af25a Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 17:39:04 +0100 Subject: [PATCH 43/80] Set log level for gunicorn --- bigchaindb/commands/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index b67c8ee8..cf8ddb4f 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -35,7 +35,10 @@ def configure_bigchaindb(command): @functools.wraps(command) def configure(args): try: - config_from_cmdline = {'log': {'level_console': args.log_level}} + config_from_cmdline = { + 'log': {'level_console': args.log_level}, + 'server': {'loglevel': args.log_level}, + } except AttributeError: config_from_cmdline = None bigchaindb.config_utils.autoconfigure( From cea78b3ae2aa3db943027788c3dd2e32e1f2490a Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 23 Mar 2017 15:28:01 +0100 Subject: [PATCH 44/80] Integrate gunicorn logs with bigchaindb logs Closes #1329 --- bigchaindb/__init__.py | 1 + bigchaindb/log/configs.py | 5 +++++ bigchaindb/log/loggers.py | 32 ++++++++++++++++++++++++++++++++ bigchaindb/log/setup.py | 9 +++++++-- tests/log/test_loggers.py | 18 ++++++++++++++++++ tests/test_config_utils.py | 1 + 6 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 bigchaindb/log/loggers.py create mode 100644 tests/log/test_loggers.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index c0e4fd56..00085314 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -36,6 +36,7 @@ config = { 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 + 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': _database_map[ os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') diff --git a/bigchaindb/log/configs.py b/bigchaindb/log/configs.py index 1be5c485..9dac0dcb 100644 --- a/bigchaindb/log/configs.py +++ b/bigchaindb/log/configs.py @@ -1,7 +1,12 @@ import logging +from logging.handlers import DEFAULT_TCP_LOGGING_PORT from os.path import expanduser, join +DEFAULT_SOCKET_LOGGING_HOST = 'localhost' +DEFAULT_SOCKET_LOGGING_PORT = DEFAULT_TCP_LOGGING_PORT +DEFAULT_SOCKET_LOGGING_ADDR = (DEFAULT_SOCKET_LOGGING_HOST, + DEFAULT_SOCKET_LOGGING_PORT) DEFAULT_LOG_DIR = expanduser('~') PUBLISHER_LOGGING_CONFIG = { diff --git a/bigchaindb/log/loggers.py b/bigchaindb/log/loggers.py new file mode 100644 index 00000000..f8c18320 --- /dev/null +++ b/bigchaindb/log/loggers.py @@ -0,0 +1,32 @@ +import logging.handlers + +from gunicorn.glogging import Logger + +from .configs import DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT + + +class HttpServerLogger(Logger): + """Custom logger class for ``gunicorn`` logs. + + Meant for internal usage only, to set the ``logger_class`` + configuration setting on gunicorn. + + """ + def setup(self, cfg): + """Setup the gunicorn access and error loggers. This overrides + the parent method. Its main goal is to simply pipe all the logs to + the TCP socket used througout BigchainDB. + + Args: + cfg (:obj:`gunicorn.config.Config`): Gunicorn configuration + object. *Ignored*. + + """ + self._set_socklog_handler(self.error_log) + self._set_socklog_handler(self.access_log) + + def _set_socklog_handler(self, log): + socket_handler = logging.handlers.SocketHandler( + DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT) + socket_handler._gunicorn = True + log.addHandler(socket_handler) diff --git a/bigchaindb/log/setup.py b/bigchaindb/log/setup.py index fdf8e49b..f3e8f7a3 100644 --- a/bigchaindb/log/setup.py +++ b/bigchaindb/log/setup.py @@ -9,7 +9,12 @@ import struct import sys from multiprocessing import Process -from .configs import PUBLISHER_LOGGING_CONFIG, SUBSCRIBER_LOGGING_CONFIG +from .configs import ( + DEFAULT_SOCKET_LOGGING_HOST, + DEFAULT_SOCKET_LOGGING_PORT, + PUBLISHER_LOGGING_CONFIG, + SUBSCRIBER_LOGGING_CONFIG, +) from bigchaindb.common.exceptions import ConfigurationError @@ -23,7 +28,7 @@ def _normalize_log_level(level): def setup_pub_logger(): dictConfig(PUBLISHER_LOGGING_CONFIG) socket_handler = logging.handlers.SocketHandler( - 'localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) + DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT) socket_handler.setLevel(logging.DEBUG) logger = logging.getLogger() logger.addHandler(socket_handler) diff --git a/tests/log/test_loggers.py b/tests/log/test_loggers.py new file mode 100644 index 00000000..795de046 --- /dev/null +++ b/tests/log/test_loggers.py @@ -0,0 +1,18 @@ +from logging.handlers import SocketHandler + + +class TestHttpServerLogger: + + def test_init(self, mocker): + from bigchaindb.log.configs import ( + DEFAULT_SOCKET_LOGGING_ADDR as expected_socket_address) + from bigchaindb.log.loggers import HttpServerLogger + mocked_config = mocker.patch( + 'gunicorn.config.Config', autospec=True, spec_set=True) + logger = HttpServerLogger(mocked_config.return_value) + assert len(logger.access_log.handlers) == 1 + assert len(logger.error_log.handlers) == 1 + assert isinstance(logger.access_log.handlers[0], SocketHandler) + assert isinstance(logger.error_log.handlers[0], SocketHandler) + assert logger.access_log.handlers[0].address == expected_socket_address + assert logger.error_log.handlers[0].address == expected_socket_address diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 4234e242..2e843914 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -195,6 +195,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'bind': SERVER_BIND, 'workers': None, 'threads': None, + 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': database, 'keypair': { From f5a32e35c5f2bdea89fcf79cb821e6f694b050f7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 23 Mar 2017 18:21:55 +0100 Subject: [PATCH 45/80] docs: 1st draft of page about updating all s/w on a BDB node on k8s --- .../cloud-deployment-templates/index.rst | 2 + .../upgrade-on-kubernetes.rst | 105 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 837dc66d..28ac7923 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -16,3 +16,5 @@ If you find the cloud deployment templates for nodes helpful, then you may also template-kubernetes-azure node-on-kubernetes add-node-on-kubernetes + upgrade-on-kubernetes + \ No newline at end of file diff --git a/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst new file mode 100644 index 00000000..348abf22 --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst @@ -0,0 +1,105 @@ +Kubernetes Template: Upgrade all Software in a BigchainDB Node +============================================================== + +This page outlines how to upgrade all the software associated +with a BigchainDB node running on Kubernetes, +including host operating systems, Docker, Kubernetes, +and BigchainDB-related software. + + +Upgrade Host OS, Docker and Kubernetes +-------------------------------------- + +Some Kubernetes installation & management systems +can do full or partial upgrades of host OSes, Docker, +or Kubernetes, e.g. +`Tectonic `_, +`Rancher `_, +and +`Kubo `_. +Consult the documentation for your system. + +**Azure Container Service (ACS).** +On Dec. 15, 2016, a Microsoft employee +`wrote `_: +"In the coming months we [the Azure Kubernetes team] will be building managed updates in the ACS service." +At the time of writing, managed updates were not yet available, +but you should check the latest +`ACS documentation `_ +to see what's available now. +Also at the time of writing, ACS only supported Ubuntu +as the host (master and agent) operating system. +You can upgrade Ubuntu and Docker on Azure +by SSHing into each of the hosts, +as documented on +:ref:`another page `. + +In general, you can SSH to each host in your Kubernetes Cluster +to update the OS and Docker. + +.. note:: + + Once you are in an SSH session with a host, + the ``docker info`` command is a handy way to detemine the + host OS (including version) and the Docker version. + +When you want to upgrade the software on a Kubernetes node, +you should "drain" the node first, +i.e. tell Kubernetes to gracefully terminate all pods +on the node and mark it as unscheduleable +(so no new pods get put on the node during its downtime). + +.. code:: + + kubectl drain $NODENAME + +There are `more details in the Kubernetes docs `_, +including instructions to make the node scheduleable again. + +To manually upgrade the host OS, +see the docs for that OS. + +To manually upgrade Docker, see +`the Docker docs `_. + +To manually upgrade all Kubernetes software in your Kubernetes cluster, see +`the Kubernetes docs `_. + + +Upgrade BigchainDB-Related Software +----------------------------------- + +We use Kubernetes "Deployments" for NGINX, BigchainDB, +and most other BigchainDB-related software. +The only exception is MongoDB; we use a Kubernetes +StatefulSet for that. + +The nice thing about Kubernetes Deployments +is that Kubernetes can manage most of the upgrade process. +A typical upgrade workflow for a single Deployment would be: + +.. code:: + + $ KUBE_EDITOR=nano kubectl edit deployment/ + +The `kubectl edit `_ +command opens the specified editor (nano in the above example), +allowing you to edit the specified Deployment *in the Kubernetes cluster*. +You can change the version tag on the Docker image, for example. +Don't forget to save your edits before exiting the editor. +The Kubernetes docs have more information about +`updating a Deployment `_. + + +The upgrade story for the MongoDB StatefulSet is *different*. +(This is because MongoDB has persistent state, +which is stored in some storage associated with a PersistentVolumeClaim.) +At the time of writing, StatefulSets were still in beta, +and they did not support automated image upgrade (Docker image tag upgrade). +We expect that to change. +Rather than trying to keep these docs up-to-date, +we advise you to check out the current +`Kubernetes docs about updating containers in StatefulSets +`_. + + From c61e3333d0e37405dcb2a7d2f30e46672bb44fa0 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 24 Mar 2017 10:57:56 +0100 Subject: [PATCH 46/80] improved readthedocs.org instructions in Release_Process.md --- Release_Process.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index ec51ceaf..22572837 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -45,10 +45,16 @@ These steps are common between minor and patch releases: 1. Make sure your local Git is in the same state as the release: e.g. `git fetch ` and `git checkout v0.9.1` 1. Make sure you have a `~/.pypirc` file containing credentials for PyPI 1. Do a `make release` to build and publish the new `bigchaindb` package on PyPI -1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. - Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is - "Active" and "Public", and make sure the new version's branch - (without the 'v' in front) is _not_ active -1. Also in readthedocs.org, go to Admin --> Advanced Settings - and make sure that "Default branch:" (i.e. what "latest" points to) - is set to the new release's tag, e.g. `v0.9.1`. (Don't miss the 'v' in front.) +1. [Login to readthedocs.org](https://readthedocs.org/accounts/login/) + as a maintainer of the BigchainDB Server docs, and: + - Go to Admin --> Advanced Settings + and make sure that "Default branch:" (i.e. what "latest" points to) + is set to the new release's tag, e.g. `v0.9.1`. + (Don't miss the 'v' in front.) + - Go to Admin --> Versions + and under **Choose Active Versions**, do these things: + 1. Make sure that the new version's tag is "Active" and "Public" + 2. Make sure the new version's branch + (without the 'v' in front) is _not_ active. + 3. Make sure the **stable** branch is _not_ active. + 4. Scroll to the bottom of the page and click the Submit button. From 0edb1c18f2c5f678cba86ad5c91a30502e2c7c6d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 24 Mar 2017 11:56:35 +0100 Subject: [PATCH 47/80] Keep gunicorn logger_class internal closes #1334 --- bigchaindb/__init__.py | 1 - bigchaindb/web/server.py | 1 + tests/test_config_utils.py | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 00085314..c0e4fd56 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -36,7 +36,6 @@ config = { 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 - 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': _database_map[ os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index bcd44d11..b1525f9f 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -88,6 +88,7 @@ def create_server(settings): if not settings.get('threads'): settings['threads'] = (multiprocessing.cpu_count() * 2) + 1 + settings['logger_class'] = 'bigchaindb.log.loggers.HttpServerLogger' app = create_app(debug=settings.get('debug', False), threads=settings['threads']) standalone = StandaloneApplication(app, settings) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 2e843914..4234e242 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -195,7 +195,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'bind': SERVER_BIND, 'workers': None, 'threads': None, - 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': database, 'keypair': { From 58d80e9731333eced2bbd513cf9f71f4281fc6bf Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Mon, 27 Mar 2017 10:43:40 +0200 Subject: [PATCH 48/80] Fix ssl param default value --- bigchaindb/backend/connection.py | 5 ++--- bigchaindb/backend/mongodb/connection.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 56b5cd82..b717703b 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None, ssl=False, login=None, password=None): + connection_timeout=None, replicaset=None, ssl=None, login=None, password=None): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -50,8 +50,7 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, # to handle these these additional args. In case of RethinkDBConnection # it just does not do anything with it. replicaset = replicaset or bigchaindb.config['database'].get('replicaset') - ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ - else ssl + ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False) login = login or bigchaindb.config['database'].get('login') password = password or bigchaindb.config['database'].get('password') diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 8b30b2db..5c54470a 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, ssl=False, login=None, password=None, **kwargs): + def __init__(self, replicaset=None, ssl=None, login=None, password=None, **kwargs): """Create a new Connection instance. Args: @@ -28,8 +28,7 @@ class MongoDBConnection(Connection): super().__init__(**kwargs) self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] - self.ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ - else ssl + self.ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False) self.login = login or bigchaindb.config['database'].get('login') self.password = password or bigchaindb.config['database'].get('password') From 441ad914cf854ba64769ed3ce1bf5dd911fc9e24 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 11:24:16 +0200 Subject: [PATCH 49/80] Improve test coverage --- tests/backend/mongodb/test_connection.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index e0b161b0..3edc31b1 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -99,6 +99,18 @@ def test_connection_run_errors(mock_client, mock_init_repl_set): assert query.run.call_count == 1 +@mock.patch('pymongo.database.Database.authenticate') +def test_connection_with_credentials(mock_authenticate): + import bigchaindb + from bigchaindb.backend.mongodb.connection import MongoDBConnection + conn = MongoDBConnection(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port'], + login='theplague', + password='secret') + conn.connect() + assert mock_authenticate.call_count == 2 + + def test_check_replica_set_not_enabled(mongodb_connection): from bigchaindb.backend.mongodb.connection import _check_replica_set from bigchaindb.common.exceptions import ConfigurationError From 9679561d89c0c0fcf22c67dfb9b9a83737c5543d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 28 Mar 2017 12:14:50 +0200 Subject: [PATCH 50/80] Update pytest command in Makefile Use new command "pytest", add verbosity, and distribute tests across available CPU cores --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7fc9c1c0..afa992bf 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,7 @@ lint: ## check style with flake8 flake8 bigchaindb tests test: ## run tests quickly with the default Python - py.test - + pytest -v -n auto test-all: ## run tests on every Python version with tox tox From 1083e04dd5eaad5289470fe478a37e03f32ec3ee Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 28 Mar 2017 12:17:35 +0200 Subject: [PATCH 51/80] Fix Makefile (test) coverage target --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index afa992bf..37bf6db8 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,8 @@ test-all: ## run tests on every Python version with tox tox coverage: ## check code coverage quickly with the default Python - coverage run --source bigchaindb py.test - - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html + pytest -v -n auto --cov=bigchaindb --cov-report term --cov-report html + $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs $(MAKE) -C docs/root clean From 699e615d47bb6b24f4d373c8edfd3a84d4b6e0c1 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 14:51:02 +0200 Subject: [PATCH 52/80] Add ssl, login, and passwd to configure command --- bigchaindb/__init__.py | 36 +++++++++++++++++++++++++++------ bigchaindb/commands/bigchain.py | 3 ++- tests/test_config_utils.py | 12 +++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 1df2551c..53e7fd2b 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,24 +5,48 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 -_database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + +_base_database_rethinkdb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'connection_timeout': 5000, - 'max_tries': 3, } -_database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), +# This might sound excessive, but having an order on the keys will +# stress users (and us) less. +_base_database_rethinkdb_keys = ('host', 'port', 'name') + +_base_database_mongodb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), + 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), + 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), + 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') +} + +_base_database_mongodb_keys = ('host', 'port', 'name', 'replicaset', + 'ssl', 'login', 'password') + +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'connection_timeout': 5000, 'max_tries': 3, } +_database_rethinkdb.update(_base_database_rethinkdb) + +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'connection_timeout': 5000, + 'max_tries': 3, +} +_database_mongodb.update(_base_database_mongodb) + +_database_keys_map = { + 'mongodb': _base_database_mongodb_keys, + 'rethinkdb': _base_database_rethinkdb_keys +} _database_map = { 'mongodb': _database_mongodb, diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index efefa9d7..a5ec9c6a 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -90,6 +90,7 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend), file=sys.stderr) + database_keys = bigchaindb._database_keys_map[args.backend] conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: @@ -99,7 +100,7 @@ def run_configure(args, skip_if_exists=False): input_on_stderr('API Server {}? (default `{}`): '.format(key, val)) \ or val - for key in ('host', 'port', 'name'): + for key in database_keys: val = conf['database'][key] conf['database'][key] = \ input_on_stderr('Database {}? (default `{}`): '.format(key, val)) \ diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 0fa5135b..d81f5f75 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -19,6 +19,15 @@ def clean_config(monkeypatch, request): monkeypatch.setattr('bigchaindb.config', original_config) +def test_ordered_keys_match_database_config(): + import bigchaindb + + assert set(bigchaindb._base_database_rethinkdb.keys()) ==\ + set(bigchaindb._base_database_rethinkdb_keys) + assert set(bigchaindb._base_database_mongodb.keys()) ==\ + set(bigchaindb._base_database_mongodb_keys) + + def test_bigchain_instance_is_initialized_when_conf_provided(request): import bigchaindb from bigchaindb import config_utils @@ -181,6 +190,9 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'connection_timeout': 5000, 'max_tries': 3, 'replicaset': 'bigchain-rs', + 'ssl': False, + 'login': None, + 'password': None } database = {} From 047108046acd57dcd6bbddd344d5a213def1d985 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 15:01:10 +0200 Subject: [PATCH 53/80] Revert "Add ssl, login, and passwd to configure command" This reverts commit 699e615d47bb6b24f4d373c8edfd3a84d4b6e0c1. --- bigchaindb/__init__.py | 36 ++++++--------------------------- bigchaindb/commands/bigchain.py | 3 +-- tests/test_config_utils.py | 12 ----------- 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 53e7fd2b..1df2551c 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,48 +5,24 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 - -_base_database_rethinkdb = { +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), + 'connection_timeout': 5000, + 'max_tries': 3, } -# This might sound excessive, but having an order on the keys will -# stress users (and us) less. -_base_database_rethinkdb_keys = ('host', 'port', 'name') - -_base_database_mongodb = { +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), - 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), - 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') -} - -_base_database_mongodb_keys = ('host', 'port', 'name', 'replicaset', - 'ssl', 'login', 'password') - -_database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'connection_timeout': 5000, 'max_tries': 3, } -_database_rethinkdb.update(_base_database_rethinkdb) - -_database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), - 'connection_timeout': 5000, - 'max_tries': 3, -} -_database_mongodb.update(_base_database_mongodb) - -_database_keys_map = { - 'mongodb': _base_database_mongodb_keys, - 'rethinkdb': _base_database_rethinkdb_keys -} _database_map = { 'mongodb': _database_mongodb, diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index a5ec9c6a..efefa9d7 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -90,7 +90,6 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend), file=sys.stderr) - database_keys = bigchaindb._database_keys_map[args.backend] conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: @@ -100,7 +99,7 @@ def run_configure(args, skip_if_exists=False): input_on_stderr('API Server {}? (default `{}`): '.format(key, val)) \ or val - for key in database_keys: + for key in ('host', 'port', 'name'): val = conf['database'][key] conf['database'][key] = \ input_on_stderr('Database {}? (default `{}`): '.format(key, val)) \ diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index d81f5f75..0fa5135b 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -19,15 +19,6 @@ def clean_config(monkeypatch, request): monkeypatch.setattr('bigchaindb.config', original_config) -def test_ordered_keys_match_database_config(): - import bigchaindb - - assert set(bigchaindb._base_database_rethinkdb.keys()) ==\ - set(bigchaindb._base_database_rethinkdb_keys) - assert set(bigchaindb._base_database_mongodb.keys()) ==\ - set(bigchaindb._base_database_mongodb_keys) - - def test_bigchain_instance_is_initialized_when_conf_provided(request): import bigchaindb from bigchaindb import config_utils @@ -190,9 +181,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'connection_timeout': 5000, 'max_tries': 3, 'replicaset': 'bigchain-rs', - 'ssl': False, - 'login': None, - 'password': None } database = {} From c6de90fa79c080bb1403260b36f873e0ed1b4a69 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 29 Mar 2017 09:56:41 +0200 Subject: [PATCH 54/80] Upgrade rapidjson to latest 0.0.11 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a38bb1f..5fb201f4 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ install_requires = [ 'pymongo~=3.4', 'pysha3~=1.0.2', 'cryptoconditions>=0.5.0', - 'python-rapidjson==0.0.8', + 'python-rapidjson==0.0.11', 'logstats>=0.2.1', 'flask>=0.10.1', 'flask-restful~=0.3.0', From ead832a130ed2136dac7735e502af5b708468b3a Mon Sep 17 00:00:00 2001 From: morrme Date: Mon, 27 Mar 2017 05:13:29 -0500 Subject: [PATCH 55/80] added Python 3.6 per issue #1331 --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index da7ae05f..e558d154 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ cache: pip python: - 3.4 - 3.5 - + - 3.6 + env: - TOXENV=flake8 - TOXENV=docsroot @@ -30,6 +31,12 @@ matrix: env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb - python: 3.5 env: BIGCHAINDB_DATABASE_BACKEND=mongodb + - python: 3.6 + addons: + rethinkdb: '2.3.5' + env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb + - python: 3.6 + env: BIGCHAINDB_DATABASE_BACKEND=mongodb before_install: sudo .ci/travis-before-install.sh From 5c2bab078fac8bdd65a3ec77a8e524084fff0c70 Mon Sep 17 00:00:00 2001 From: morrme Date: Mon, 27 Mar 2017 05:14:40 -0500 Subject: [PATCH 56/80] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d2cd2a2c..8f299471 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = true -envlist = py{34,35}-{rethinkdb,mongodb}, flake8, docsroot, docsserver +envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver [base] basepython = python3.5 From 4bcd7dd1e2a80feb56d5e0fd4d93f1879b6f9ae2 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 30 Mar 2017 14:56:33 +0200 Subject: [PATCH 57/80] Delete outdated speed-tests folder --- speed-tests/README.md | 3 -- speed-tests/speed_tests.py | 97 -------------------------------------- 2 files changed, 100 deletions(-) delete mode 100644 speed-tests/README.md delete mode 100644 speed-tests/speed_tests.py diff --git a/speed-tests/README.md b/speed-tests/README.md deleted file mode 100644 index 7b07d338..00000000 --- a/speed-tests/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Speed Tests - -This folder contains tests related to the code performance of a single node. \ No newline at end of file diff --git a/speed-tests/speed_tests.py b/speed-tests/speed_tests.py deleted file mode 100644 index 87a81b0f..00000000 --- a/speed-tests/speed_tests.py +++ /dev/null @@ -1,97 +0,0 @@ -import json -import time - -import rapidjson -from line_profiler import LineProfiler - -import bigchaindb - -# BIG TODO: Adjust for new transaction model - - -def speedtest_validate_transaction(): - # create a transaction - b = bigchaindb.Bigchain() - tx = b.create_transaction(b.me, b.me, None, 'CREATE') - tx_signed = b.sign_transaction(tx, b.me_private) - - # setup the profiler - profiler = LineProfiler() - profiler.enable_by_count() - profiler.add_function(bigchaindb.Bigchain.validate_transaction) - - # validate_transaction 1000 times - for i in range(1000): - b.validate_transaction(tx_signed) - - profiler.print_stats() - - -def speedtest_serialize_block_json(): - # create a block - b = bigchaindb.Bigchain() - tx = b.create_transaction(b.me, b.me, None, 'CREATE') - tx_signed = b.sign_transaction(tx, b.me_private) - block = b.create_block([tx_signed] * 1000) - - time_start = time.time() - for _ in range(1000): - _ = json.dumps(block, skipkeys=False, ensure_ascii=False, sort_keys=True) - time_elapsed = time.time() - time_start - - print('speedtest_serialize_block_json: {} s'.format(time_elapsed)) - - -def speedtest_serialize_block_rapidjson(): - # create a block - b = bigchaindb.Bigchain() - tx = b.create_transaction(b.me, b.me, None, 'CREATE') - tx_signed = b.sign_transaction(tx, b.me_private) - block = b.create_block([tx_signed] * 1000) - - time_start = time.time() - for _ in range(1000): - _ = rapidjson.dumps(block, skipkeys=False, ensure_ascii=False, sort_keys=True) - time_elapsed = time.time() - time_start - - print('speedtest_serialize_block_rapidjson: {} s'.format(time_elapsed)) - - -def speedtest_deserialize_block_json(): - # create a block - b = bigchaindb.Bigchain() - tx = b.create_transaction(b.me, b.me, None, 'CREATE') - tx_signed = b.sign_transaction(tx, b.me_private) - block = b.create_block([tx_signed] * 1000) - block_serialized = json.dumps(block, skipkeys=False, ensure_ascii=False, sort_keys=True) - - time_start = time.time() - for _ in range(1000): - _ = json.loads(block_serialized) - time_elapsed = time.time() - time_start - - print('speedtest_deserialize_block_json: {} s'.format(time_elapsed)) - - -def speedtest_deserialize_block_rapidjson(): - # create a block - b = bigchaindb.Bigchain() - tx = b.create_transaction(b.me, b.me, None, 'CREATE') - tx_signed = b.sign_transaction(tx, b.me_private) - block = b.create_block([tx_signed] * 1000) - block_serialized = rapidjson.dumps(block, skipkeys=False, ensure_ascii=False, sort_keys=True) - - time_start = time.time() - for _ in range(1000): - _ = rapidjson.loads(block_serialized) - time_elapsed = time.time() - time_start - - print('speedtest_deserialize_block_rapidjson: {} s'.format(time_elapsed)) - - -if __name__ == '__main__': - speedtest_validate_transaction() - speedtest_serialize_block_json() - speedtest_serialize_block_rapidjson() - speedtest_deserialize_block_json() - speedtest_deserialize_block_rapidjson() From 64ef0dd9a17e70b3e02b7a2c5f67eda03073e13d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 31 Mar 2017 10:54:02 +0200 Subject: [PATCH 58/80] Test docs building and flake8 only for Python 3.6 --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e558d154..9fc4e278 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,17 @@ env: matrix: fast_finish: true exclude: - - python: 3.4 + - python: 3.4 env: TOXENV=flake8 - - python: 3.4 + - python: 3.4 env: TOXENV=docsroot - - python: 3.4 + - python: 3.4 + env: TOXENV=docsserver + - python: 3.5 + env: TOXENV=flake8 + - python: 3.5 + env: TOXENV=docsroot + - python: 3.5 env: TOXENV=docsserver include: - python: 3.4 From 36edbc5f354e1b5b62af03246e60e546951f82a8 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 31 Mar 2017 10:54:59 +0200 Subject: [PATCH 59/80] Set base version for Python to 3.6 in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8f299471..bdaea034 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ skipsdist = true envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver [base] -basepython = python3.5 +basepython = python3.6 deps = pip>=9.0.1 [testenv] From 6fb793e52f716a356579fc60349840fe6e2bd282 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 29 Mar 2017 10:22:15 +0200 Subject: [PATCH 60/80] Update changelog for 0.9.5 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9af7ccc3..2148903b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [0.9.5] - 2017-03-29 +Tag name: v0.9.5 + +### Fixed +Upgrade `python-rapidjson` to `0.0.11`(fixes #1350 - thanks to @ferOnti for +reporting). + ## [0.9.4] - 2017-03-16 Tag name: v0.9.4 From 87eb070ed68533abd42f736b3ac293e0c6883416 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 3 Mar 2017 01:47:08 +0100 Subject: [PATCH 61/80] Refactor core.BigchainDB.get_outputs --- bigchaindb/core.py | 64 +++++++++++++++++++++++++-------------------- bigchaindb/utils.py | 13 +++++++++ 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index a9143f33..e6783a6d 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -19,14 +19,17 @@ class Bigchain(object): Create, read, sign, write transactions to the database """ - # return if a block has been voted invalid BLOCK_INVALID = 'invalid' - # return if a block is valid, or tx is in valid block + """return if a block has been voted invalid""" + BLOCK_VALID = TX_VALID = 'valid' - # return if block is undecided, or tx is in undecided block + """return if a block is valid, or tx is in valid block""" + BLOCK_UNDECIDED = TX_UNDECIDED = 'undecided' - # return if transaction is in backlog + """return if block is undecided, or tx is in undecided block""" + TX_IN_BACKLOG = 'backlog' + """return if transaction is in backlog""" def __init__(self, public_key=None, private_key=None, keyring=[], connection=None, backlog_reassign_delay=None): """Initialize the Bigchain instance @@ -372,32 +375,37 @@ class Bigchain(object): """ # get all transactions in which owner is in the `owners_after` list response = backend.query.get_owned_ids(self.connection, owner) - links = [] + return [ + TransactionLink(tx['id'], index) + for tx in response + if not self.is_tx_strictly_in_invalid_block(tx['id']) + for index, output in enumerate(tx['outputs']) + if utils.output_has_owner(output, owner) + ] - for tx in response: - # disregard transactions from invalid blocks - validity = self.get_blocks_status_containing_tx(tx['id']) - if Bigchain.BLOCK_VALID not in validity.values(): - if Bigchain.BLOCK_UNDECIDED not in validity.values(): - continue + def is_tx_strictly_in_invalid_block(self, txid): + """ + Checks whether the transaction with the given ``txid`` + *strictly* belongs to an invalid block. - # NOTE: It's OK to not serialize the transaction here, as we do not - # use it after the execution of this function. - # a transaction can contain multiple outputs so we need to iterate over all of them - # to get a list of outputs available to spend - for index, output in enumerate(tx['outputs']): - # for simple signature conditions there are no subfulfillments - # check if the owner is in the condition `owners_after` - if len(output['public_keys']) == 1: - if output['condition']['details']['public_key'] == owner: - links.append(TransactionLink(tx['id'], index)) - else: - # for transactions with multiple `public_keys` there will be several subfulfillments nested - # in the condition. We need to iterate the subfulfillments to make sure there is a - # subfulfillment for `owner` - if utils.condition_details_has_owner(output['condition']['details'], owner): - links.append(TransactionLink(tx['id'], index)) - return links + Args: + txid (str): Transaction id. + + Returns: + bool: ``True`` if the transaction *strictly* belongs to a + block that is invalid. ``False`` otherwise. + + Note: + Since a transaction may be in multiple blocks, with + different statuses, the term "strictly" is used to + emphasize that if a transaction is said to be in an invalid + block, it means that it is not in any other block that is + either valid or undecided. + + """ + validity = self.get_blocks_status_containing_tx(txid) + return (Bigchain.BLOCK_VALID not in validity.values() and + Bigchain.BLOCK_UNDECIDED not in validity.values()) def get_owned_ids(self, owner): """Retrieve a list of ``txid`` s that can be used as inputs. diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 4d7177d9..f87916b7 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -113,6 +113,19 @@ def condition_details_has_owner(condition_details, owner): return False +def output_has_owner(output, owner): + # TODO + # Check whether it is really necessary to treat the single key case + # differently from the multiple keys case, and why not just use the same + # function for both cases. + if len(output['public_keys']) > 1: + return condition_details_has_owner( + output['condition']['details'], owner) + elif len(output['public_keys']) == 1: + return output['condition']['details']['public_key'] == owner + # TODO raise proper exception, e.g. invalid tx payload? + + def is_genesis_block(block): """Check if the block is the genesis block. From ca200b1da7ba77f9aa753fd20b8eb09ea4a95838 Mon Sep 17 00:00:00 2001 From: Jack Riches Date: Sun, 2 Apr 2017 12:22:56 +0100 Subject: [PATCH 62/80] Treat --log-level argument as case-insensitive --- bigchaindb/commands/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index cf8ddb4f..6cc5cb6a 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -198,6 +198,7 @@ base_parser.add_argument('-c', '--config', '(use "-" for stdout)') base_parser.add_argument('-l', '--log-level', + type=lambda l: l.upper(), # case insensitive conversion choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='Log level') From 09866920af8ae5b197dc45eb976ebf2c0cb0d1bd Mon Sep 17 00:00:00 2001 From: Anuj Date: Sun, 2 Apr 2017 17:53:39 +0530 Subject: [PATCH 63/80] Pretty message when dropping a non-existent database --- bigchaindb/commands/bigchain.py | 8 ++++++-- tests/commands/test_commands.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index be17d75f..ce0cbfa0 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -12,7 +12,8 @@ import sys from bigchaindb.common import crypto from bigchaindb.common.exceptions import (StartupError, DatabaseAlreadyExists, - KeypairNotFoundException) + KeypairNotFoundException, + DatabaseDoesNotExist) import bigchaindb from bigchaindb import backend, processes from bigchaindb.backend import schema @@ -166,7 +167,10 @@ def run_drop(args): conn = backend.connect() dbname = bigchaindb.config['database']['name'] - schema.drop_database(conn, dbname) + try: + schema.drop_database(conn, dbname) + except DatabaseDoesNotExist: + print("Cannot drop '{name}'. The database does not exist.".format(name=dbname), file=sys.stderr) @configure_bigchaindb diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 50b995b0..ad603351 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -149,6 +149,18 @@ def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): assert mock_db_drop.called +@patch('bigchaindb.backend.schema.drop_database') +def test_drop_db_when_db_does_not_exist(mock_db_drop, capsys): + from bigchaindb.commands.bigchain import run_drop + from bigchaindb.common.exceptions import DatabaseDoesNotExist + args = Namespace(config=None, yes=True) + mock_db_drop.side_effect = DatabaseDoesNotExist + + run_drop(args) + output_message = capsys.readouterr()[1] + assert output_message == "Cannot drop 'bigchain'. The database does not exist.\n" + + @patch('bigchaindb.backend.schema.drop_database') def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch): from bigchaindb.commands.bigchain import run_drop From f3f1ecdaecf2e42f713e6090176eeb24f6b074e3 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sun, 2 Apr 2017 16:46:41 +0200 Subject: [PATCH 64/80] Added to HOW_TO_HANDLE_PULL_REQUESTS.md Added new subsection: How to Handle CLA Agreement Emails with No Associated Pull Request --- HOW_TO_HANDLE_PULL_REQUESTS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/HOW_TO_HANDLE_PULL_REQUESTS.md b/HOW_TO_HANDLE_PULL_REQUESTS.md index 4dfbec15..6114c7ac 100644 --- a/HOW_TO_HANDLE_PULL_REQUESTS.md +++ b/HOW_TO_HANDLE_PULL_REQUESTS.md @@ -51,3 +51,15 @@ END BLOCK (END OF EMAIL) The next step is to wait for them to copy that comment into the comments of the indicated pull request. Once they do so, it's safe to merge the pull request. + +## How to Handle CLA Agreement Emails with No Associated Pull Request + +Reply with an email like this: + +Hi [First Name], + +Today I got an email (copied below) to tell me that you agreed to the BigchainDB Contributor License Agreement. Did you intend to do that? + +If no, then you can ignore this email. + +If yes, then there's another step to connect your email address with your GitHub account. To do that, you must first create a pull request in one of the BigchainDB repositories on GitHub. Once you've done that, please reply to this email with a link to the pull request. Then I'll send you a special block of text to paste into the comments on that pull request. From eff1406c09a6f05eb7d9c97fcf840f8daa8d29d8 Mon Sep 17 00:00:00 2001 From: Jack Riches Date: Sun, 2 Apr 2017 23:46:44 +0100 Subject: [PATCH 65/80] Refactor awawy unnecessary lambda to str.upper --- bigchaindb/commands/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 6cc5cb6a..15887340 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -198,7 +198,7 @@ base_parser.add_argument('-c', '--config', '(use "-" for stdout)') base_parser.add_argument('-l', '--log-level', - type=lambda l: l.upper(), # case insensitive conversion + type=str.upper # convert to uppercase for comparison to choices choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='Log level') From c3f89fd447e7729dad98b521c5c484311e1e2a8a Mon Sep 17 00:00:00 2001 From: Anuj Date: Mon, 3 Apr 2017 13:13:22 +0530 Subject: [PATCH 66/80] Taking DB name from config in test for non-existent db drop --- tests/commands/test_commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index ad603351..c0e2b5af 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -151,6 +151,7 @@ def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): @patch('bigchaindb.backend.schema.drop_database') def test_drop_db_when_db_does_not_exist(mock_db_drop, capsys): + from bigchaindb import config from bigchaindb.commands.bigchain import run_drop from bigchaindb.common.exceptions import DatabaseDoesNotExist args = Namespace(config=None, yes=True) @@ -158,7 +159,8 @@ def test_drop_db_when_db_does_not_exist(mock_db_drop, capsys): run_drop(args) output_message = capsys.readouterr()[1] - assert output_message == "Cannot drop 'bigchain'. The database does not exist.\n" + assert output_message == "Cannot drop '{name}'. The database does not exist.\n".format( + name=config['database']['name']) @patch('bigchaindb.backend.schema.drop_database') From e7b0b227f18f88747a992e29367d52d20167185f Mon Sep 17 00:00:00 2001 From: Lavina Date: Wed, 29 Mar 2017 20:05:01 +0530 Subject: [PATCH 67/80] Rename bigchain.py command module to bigchaindb.py --- .../commands/{bigchain.py => bigchaindb.py} | 0 docs/server/source/appendices/commands.rst | 4 +- setup.py | 2 +- tests/commands/conftest.py | 8 +-- tests/commands/rethinkdb/test_commands.py | 10 ++-- tests/commands/test_commands.py | 58 +++++++++---------- tests/commands/test_utils.py | 4 +- 7 files changed, 43 insertions(+), 43 deletions(-) rename bigchaindb/commands/{bigchain.py => bigchaindb.py} (100%) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchaindb.py similarity index 100% rename from bigchaindb/commands/bigchain.py rename to bigchaindb/commands/bigchaindb.py diff --git a/docs/server/source/appendices/commands.rst b/docs/server/source/appendices/commands.rst index 35d37b27..fd367cdb 100644 --- a/docs/server/source/appendices/commands.rst +++ b/docs/server/source/appendices/commands.rst @@ -6,10 +6,10 @@ Command Line Interface :special-members: __init__ -:mod:`bigchaindb.commands.bigchain` +:mod:`bigchaindb.commands.bigchaindb` ----------------------------------- -.. automodule:: bigchaindb.commands.bigchain +.. automodule:: bigchaindb.commands.bigchaindb :mod:`bigchaindb.commands.utils` diff --git a/setup.py b/setup.py index 5fb201f4..c05b554a 100644 --- a/setup.py +++ b/setup.py @@ -117,7 +117,7 @@ setup( entry_points={ 'console_scripts': [ - 'bigchaindb=bigchaindb.commands.bigchain:main' + 'bigchaindb=bigchaindb.commands.bigchaindb:main' ], }, install_requires=install_requires, diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index 30c577f5..4a60c0cc 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -5,8 +5,8 @@ import pytest @pytest.fixture def mock_run_configure(monkeypatch): - from bigchaindb.commands import bigchain - monkeypatch.setattr(bigchain, 'run_configure', lambda *args, **kwargs: None) + from bigchaindb.commands import bigchaindb + monkeypatch.setattr(bigchaindb, 'run_configure', lambda *args, **kwargs: None) @pytest.fixture @@ -17,8 +17,8 @@ def mock_write_config(monkeypatch): @pytest.fixture def mock_db_init_with_existing_db(monkeypatch): - from bigchaindb.commands import bigchain - monkeypatch.setattr(bigchain, '_run_init', lambda: None) + from bigchaindb.commands import bigchaindb + monkeypatch.setattr(bigchaindb, '_run_init', lambda: None) @pytest.fixture diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index 165fef0d..0eab914c 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -11,7 +11,7 @@ def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, mock_processes_start, mock_db_init_with_existing_db, mocked_setup_logging): - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start args = Namespace(start_rethinkdb=True, allow_temp_keypair=False, config=None, yes=True) run_start(args) @@ -39,7 +39,7 @@ def test_start_rethinkdb_exits_when_cannot_start(mock_popen): @patch('rethinkdb.ast.Table.reconfigure') def test_set_shards(mock_reconfigure, monkeypatch, b): - from bigchaindb.commands.bigchain import run_set_shards + from bigchaindb.commands.bigchaindb import run_set_shards # this will mock the call to retrieve the database config # we will set it to return one replica @@ -62,7 +62,7 @@ def test_set_shards(mock_reconfigure, monkeypatch, b): def test_set_shards_raises_exception(monkeypatch, b): - from bigchaindb.commands.bigchain import run_set_shards + from bigchaindb.commands.bigchaindb import run_set_shards # test that we are correctly catching the exception def mock_raise(*args, **kwargs): @@ -82,7 +82,7 @@ def test_set_shards_raises_exception(monkeypatch, b): @patch('rethinkdb.ast.Table.reconfigure') def test_set_replicas(mock_reconfigure, monkeypatch, b): - from bigchaindb.commands.bigchain import run_set_replicas + from bigchaindb.commands.bigchaindb import run_set_replicas # this will mock the call to retrieve the database config # we will set it to return two shards @@ -105,7 +105,7 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b): def test_set_replicas_raises_exception(monkeypatch, b): - from bigchaindb.commands.bigchain import run_set_replicas + from bigchaindb.commands.bigchaindb import run_set_replicas # test that we are correctly catching the exception def mock_raise(*args, **kwargs): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 50b995b0..186dfbc6 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -8,7 +8,7 @@ import pytest def test_make_sure_we_dont_remove_any_command(): # thanks to: http://stackoverflow.com/a/18161115/597097 - from bigchaindb.commands.bigchain import create_parser + from bigchaindb.commands.bigchaindb import create_parser parser = create_parser() @@ -27,7 +27,7 @@ def test_make_sure_we_dont_remove_any_command(): @patch('bigchaindb.commands.utils.start') def test_main_entrypoint(mock_start): - from bigchaindb.commands.bigchain import main + from bigchaindb.commands.bigchaindb import main main() assert mock_start.called @@ -37,7 +37,7 @@ def test_bigchain_run_start(mock_run_configure, mock_processes_start, mock_db_init_with_existing_db, mocked_setup_logging): - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True) run_start(args) mocked_setup_logging.assert_called_once_with(user_log_config={}) @@ -48,7 +48,7 @@ def test_bigchain_run_start_assume_yes_create_default_config( monkeypatch, mock_processes_start, mock_generate_key_pair, mock_db_init_with_existing_db, mocked_setup_logging): import bigchaindb - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start from bigchaindb import config_utils value = {} @@ -76,7 +76,7 @@ def test_bigchain_run_start_assume_yes_create_default_config( @pytest.mark.usefixtures('ignore_local_config_file') def test_bigchain_show_config(capsys): from bigchaindb import config - from bigchaindb.commands.bigchain import run_show_config + from bigchaindb.commands.bigchaindb import run_show_config args = Namespace(config=None) _, _ = capsys.readouterr() @@ -89,7 +89,7 @@ def test_bigchain_show_config(capsys): def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): from bigchaindb import config - from bigchaindb.commands.bigchain import run_export_my_pubkey + from bigchaindb.commands.bigchaindb import run_export_my_pubkey args = Namespace(config='dummy') # so in run_export_my_pubkey(args) below, @@ -108,7 +108,7 @@ def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): from bigchaindb import config - from bigchaindb.commands.bigchain import run_export_my_pubkey + from bigchaindb.commands.bigchaindb import run_export_my_pubkey args = Namespace(config='dummy') monkeypatch.setitem(config['keypair'], 'public', None) @@ -125,14 +125,14 @@ def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): def test_bigchain_run_init_when_db_exists(mock_db_init_with_existing_db): - from bigchaindb.commands.bigchain import run_init + from bigchaindb.commands.bigchaindb import run_init args = Namespace(config=None) run_init(args) @patch('bigchaindb.backend.schema.drop_database') def test_drop_db_when_assumed_yes(mock_db_drop): - from bigchaindb.commands.bigchain import run_drop + from bigchaindb.commands.bigchaindb import run_drop args = Namespace(config=None, yes=True) run_drop(args) @@ -141,9 +141,9 @@ def test_drop_db_when_assumed_yes(mock_db_drop): @patch('bigchaindb.backend.schema.drop_database') def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): - from bigchaindb.commands.bigchain import run_drop + from bigchaindb.commands.bigchaindb import run_drop args = Namespace(config=None, yes=False) - monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'y') + monkeypatch.setattr('bigchaindb.commands.bigchaindb.input_on_stderr', lambda x: 'y') run_drop(args) assert mock_db_drop.called @@ -151,16 +151,16 @@ def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): @patch('bigchaindb.backend.schema.drop_database') def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch): - from bigchaindb.commands.bigchain import run_drop + from bigchaindb.commands.bigchaindb import run_drop args = Namespace(config=None, yes=False) - monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'n') + monkeypatch.setattr('bigchaindb.commands.bigchaindb.input_on_stderr', lambda x: 'n') run_drop(args) assert not mock_db_drop.called def test_run_configure_when_config_exists_and_skipping(monkeypatch): - from bigchaindb.commands.bigchain import run_configure + from bigchaindb.commands.bigchaindb import run_configure monkeypatch.setattr('os.path.exists', lambda path: True) args = Namespace(config='foo', yes=True) return_value = run_configure(args, skip_if_exists=True) @@ -174,7 +174,7 @@ def test_run_configure_when_config_does_not_exist(monkeypatch, mock_write_config, mock_generate_key_pair, mock_bigchaindb_backup_config): - from bigchaindb.commands.bigchain import run_configure + from bigchaindb.commands.bigchaindb import run_configure monkeypatch.setattr('os.path.exists', lambda path: False) monkeypatch.setattr('builtins.input', lambda: '\n') args = Namespace(config='foo', backend='rethinkdb', yes=True) @@ -191,7 +191,7 @@ def test_run_configure_when_config_does_exist(monkeypatch, def mock_write_config(newconfig, filename=None): value['return'] = newconfig - from bigchaindb.commands.bigchain import run_configure + from bigchaindb.commands.bigchaindb import run_configure monkeypatch.setattr('os.path.exists', lambda path: True) monkeypatch.setattr('builtins.input', lambda: '\n') monkeypatch.setattr('bigchaindb.config_utils.write_config', mock_write_config) @@ -207,7 +207,7 @@ def test_run_configure_when_config_does_exist(monkeypatch, )) def test_run_configure_with_backend(backend, monkeypatch, mock_write_config): import bigchaindb - from bigchaindb.commands.bigchain import run_configure + from bigchaindb.commands.bigchaindb import run_configure value = {} @@ -238,7 +238,7 @@ def test_allow_temp_keypair_generates_one_on_the_fly( mock_gen_keypair, mock_processes_start, mock_db_init_with_existing_db, mocked_setup_logging): import bigchaindb - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start bigchaindb.config['keypair'] = {'private': None, 'public': None} @@ -258,7 +258,7 @@ def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, mock_db_init_with_existing_db, mocked_setup_logging): import bigchaindb - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start # Preconditions for the test original_private_key = bigchaindb.config['keypair']['private'] @@ -279,7 +279,7 @@ def test_run_start_when_db_already_exists(mocker, monkeypatch, run_start_args, mocked_setup_logging): - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start from bigchaindb.common.exceptions import DatabaseAlreadyExists mocked_start = mocker.patch('bigchaindb.processes.start') @@ -287,7 +287,7 @@ def test_run_start_when_db_already_exists(mocker, raise DatabaseAlreadyExists() monkeypatch.setattr( - 'bigchaindb.commands.bigchain._run_init', mock_run_init) + 'bigchaindb.commands.bigchaindb._run_init', mock_run_init) run_start(run_start_args) mocked_setup_logging.assert_called_once_with(user_log_config={}) assert mocked_start.called @@ -297,7 +297,7 @@ def test_run_start_when_keypair_not_found(mocker, monkeypatch, run_start_args, mocked_setup_logging): - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start from bigchaindb.commands.messages import CANNOT_START_KEYPAIR_NOT_FOUND from bigchaindb.common.exceptions import KeypairNotFoundException mocked_start = mocker.patch('bigchaindb.processes.start') @@ -306,7 +306,7 @@ def test_run_start_when_keypair_not_found(mocker, raise KeypairNotFoundException() monkeypatch.setattr( - 'bigchaindb.commands.bigchain._run_init', mock_run_init) + 'bigchaindb.commands.bigchaindb._run_init', mock_run_init) with pytest.raises(SystemExit) as exc: run_start(run_start_args) @@ -321,7 +321,7 @@ def test_run_start_when_start_rethinkdb_fails(mocker, monkeypatch, run_start_args, mocked_setup_logging): - from bigchaindb.commands.bigchain import run_start + from bigchaindb.commands.bigchaindb import run_start from bigchaindb.commands.messages import RETHINKDB_STARTUP_ERROR from bigchaindb.common.exceptions import StartupError run_start_args.start_rethinkdb = True @@ -348,7 +348,7 @@ def test_run_start_when_start_rethinkdb_fails(mocker, @patch('bigchaindb.commands.utils.start') def test_calling_main(start_mock, base_parser_mock, parse_args_mock, monkeypatch): - from bigchaindb.commands.bigchain import main + from bigchaindb.commands.bigchaindb import main argparser_mock = Mock() parser = Mock() @@ -404,9 +404,9 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, @pytest.mark.usefixtures('ignore_local_config_file') -@patch('bigchaindb.commands.bigchain.add_replicas') +@patch('bigchaindb.commands.bigchaindb.add_replicas') def test_run_add_replicas(mock_add_replicas): - from bigchaindb.commands.bigchain import run_add_replicas + from bigchaindb.commands.bigchaindb import run_add_replicas from bigchaindb.backend.exceptions import OperationError args = Namespace(config=None, replicas=['localhost:27017']) @@ -435,9 +435,9 @@ def test_run_add_replicas(mock_add_replicas): @pytest.mark.usefixtures('ignore_local_config_file') -@patch('bigchaindb.commands.bigchain.remove_replicas') +@patch('bigchaindb.commands.bigchaindb.remove_replicas') def test_run_remove_replicas(mock_remove_replicas): - from bigchaindb.commands.bigchain import run_remove_replicas + from bigchaindb.commands.bigchaindb import run_remove_replicas from bigchaindb.backend.exceptions import OperationError args = Namespace(config=None, replicas=['localhost:27017']) diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index 5f190717..f3b64c18 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -54,7 +54,7 @@ def test_configure_bigchaindb_logging(log_level): def test_start_raises_if_command_not_implemented(): from bigchaindb.commands import utils - from bigchaindb.commands.bigchain import create_parser + from bigchaindb.commands.bigchaindb import create_parser parser = create_parser() @@ -66,7 +66,7 @@ def test_start_raises_if_command_not_implemented(): def test_start_raises_if_no_arguments_given(): from bigchaindb.commands import utils - from bigchaindb.commands.bigchain import create_parser + from bigchaindb.commands.bigchaindb import create_parser parser = create_parser() From 57d3770564d8ef8d6665815a3d53f2d30faa6ac4 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 31 Mar 2017 13:26:19 +0200 Subject: [PATCH 68/80] Add missing underline title characters --- docs/server/source/appendices/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/appendices/commands.rst b/docs/server/source/appendices/commands.rst index fd367cdb..460145f4 100644 --- a/docs/server/source/appendices/commands.rst +++ b/docs/server/source/appendices/commands.rst @@ -7,7 +7,7 @@ Command Line Interface :mod:`bigchaindb.commands.bigchaindb` ------------------------------------ +------------------------------------- .. automodule:: bigchaindb.commands.bigchaindb From cee2f94f89f632fd853430825a683c7d92407729 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 3 Apr 2017 11:57:56 +0200 Subject: [PATCH 69/80] Remove benchmarking-tests folder. Remove references to removed folders. --- .gitattributes | 4 +- .gitignore | 2 - benchmarking-tests/README.md | 3 - benchmarking-tests/benchmark_utils.py | 154 -------------------------- benchmarking-tests/fabfile.py | 46 -------- benchmarking-tests/test1/README.md | 20 ---- codecov.yml | 2 - 7 files changed, 1 insertion(+), 230 deletions(-) delete mode 100644 benchmarking-tests/README.md delete mode 100644 benchmarking-tests/benchmark_utils.py delete mode 100644 benchmarking-tests/fabfile.py delete mode 100644 benchmarking-tests/test1/README.md diff --git a/.gitattributes b/.gitattributes index cd945c78..d278a72d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,11 +1,9 @@ -benchmarking-tests export-ignore deploy-cluster-aws export-ignore docs export-ignore ntools export-ignore -speed-tests export-ignore tests export-ignore .gitattributes export-ignore .gitignore export-ignore .travis.yml export-ignore *.md export-ignore -codecov.yml export-ignore \ No newline at end of file +codecov.yml export-ignore diff --git a/.gitignore b/.gitignore index efa00db2..7aba48d1 100644 --- a/.gitignore +++ b/.gitignore @@ -71,8 +71,6 @@ deploy-cluster-aws/confiles/ deploy-cluster-aws/client_confile deploy-cluster-aws/hostlist.py deploy-cluster-aws/ssh_key.py -benchmarking-tests/hostlist.py -benchmarking-tests/ssh_key.py # Ansible-specific files ntools/one-m/ansible/hosts diff --git a/benchmarking-tests/README.md b/benchmarking-tests/README.md deleted file mode 100644 index d94ec70b..00000000 --- a/benchmarking-tests/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Benchmarking tests - -This folder contains util files and test case folders to benchmark the performance of a BigchainDB cluster. \ No newline at end of file diff --git a/benchmarking-tests/benchmark_utils.py b/benchmarking-tests/benchmark_utils.py deleted file mode 100644 index d7418a36..00000000 --- a/benchmarking-tests/benchmark_utils.py +++ /dev/null @@ -1,154 +0,0 @@ -import multiprocessing as mp -import uuid -import argparse -import csv -import time -import logging -import rethinkdb as r - -from bigchaindb.common.transaction import Transaction - -from bigchaindb import Bigchain -from bigchaindb.utils import ProcessGroup -from bigchaindb.commands import utils - - -SIZE_OF_FILLER = {'minimal': 0, - 'small': 10**3, - 'medium': 10**4, - 'large': 10**5} - - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def create_write_transaction(tx_left, payload_filler): - b = Bigchain() - payload_dict = {} - if payload_filler: - payload_dict['filler'] = payload_filler - while tx_left > 0: - # Include a random uuid string in the payload - # to prevent duplicate transactions - # (i.e. transactions with the same hash) - payload_dict['msg'] = str(uuid.uuid4()) - tx = Transaction.create([b.me], [b.me], payload=payload_dict) - tx = tx.sign([b.me_private]) - b.write_transaction(tx) - tx_left -= 1 - - -def run_add_backlog(args): - tx_left = args.num_transactions // mp.cpu_count() - payload_filler = 'x' * SIZE_OF_FILLER[args.payload_size] - workers = ProcessGroup(target=create_write_transaction, - args=(tx_left, payload_filler)) - workers.start() - - -def run_gather_metrics(args): - # setup a rethinkdb connection - conn = r.connect(args.bigchaindb_host, 28015, 'bigchain') - - # setup csv writer - csv_file = open(args.csvfile, 'w') - csv_writer = csv.writer(csv_file) - - # query for the number of transactions on the backlog - num_transactions = r.table('backlog').count().run(conn) - num_transactions_received = 0 - initial_time = None - logger.info('Starting gathering metrics.') - logger.info('{} transasctions in the backlog'.format(num_transactions)) - logger.info('This process should exit automatically. ' - 'If this does not happen you can exit at any time using Ctrl-C ' - 'saving all the metrics gathered up to this point.') - - logger.info('\t{:<20} {:<20} {:<20} {:<20}'.format( - 'timestamp', - 'tx in block', - 'tx/s', - '% complete' - )) - - # listen to the changefeed - try: - for change in r.table('bigchain').changes().run(conn): - # check only for new blocks - if change['old_val'] is None: - block_num_transactions = len( - change['new_val']['block']['transactions'] - ) - time_now = time.time() - csv_writer.writerow( - [str(time_now), str(block_num_transactions)] - ) - - # log statistics - if initial_time is None: - initial_time = time_now - - num_transactions_received += block_num_transactions - elapsed_time = time_now - initial_time - percent_complete = round( - (num_transactions_received / num_transactions) * 100 - ) - - if elapsed_time != 0: - transactions_per_second = round( - num_transactions_received / elapsed_time - ) - else: - transactions_per_second = float('nan') - - logger.info('\t{:<20} {:<20} {:<20} {:<20}'.format( - time_now, - block_num_transactions, - transactions_per_second, - percent_complete - )) - - if (num_transactions - num_transactions_received) == 0: - break - except KeyboardInterrupt: - logger.info('Interrupted. Exiting early...') - finally: - # close files - csv_file.close() - - -def main(): - parser = argparse.ArgumentParser(description='BigchainDB benchmarking utils') - subparsers = parser.add_subparsers(title='Commands', dest='command') - - # add transactions to backlog - backlog_parser = subparsers.add_parser('add-backlog', - help='Add transactions to the backlog') - backlog_parser.add_argument('num_transactions', - metavar='num_transactions', - type=int, default=0, - help='Number of transactions to add to the backlog') - backlog_parser.add_argument('-s', '--payload-size', - choices=SIZE_OF_FILLER.keys(), - default='minimal', - help='Payload size') - - # metrics - metrics_parser = subparsers.add_parser('gather-metrics', - help='Gather metrics to a csv file') - - metrics_parser.add_argument('-b', '--bigchaindb-host', - required=True, - help=('Bigchaindb node hostname to connect ' - 'to gather cluster metrics')) - - metrics_parser.add_argument('-c', '--csvfile', - required=True, - help='Filename to save the metrics') - - utils.start(parser, globals()) - - -if __name__ == '__main__': - main() diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py deleted file mode 100644 index 0dd4e964..00000000 --- a/benchmarking-tests/fabfile.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import with_statement, unicode_literals - -from fabric.api import sudo, env, hosts -from fabric.api import task, parallel -from fabric.contrib.files import sed -from fabric.operations import run, put -from fabric.context_managers import settings - -from hostlist import public_dns_names -from ssh_key import ssh_key_path - -# Ignore known_hosts -# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts -env.disable_known_hosts = True - -# What remote servers should Fabric connect to? With what usernames? -env.user = 'ubuntu' -env.hosts = public_dns_names - -# SSH key files to try when connecting: -# http://docs.fabfile.org/en/1.10/usage/env.html#key-filename -env.key_filename = ssh_key_path - - -@task -@parallel -def put_benchmark_utils(): - put('benchmark_utils.py') - - -@task -@parallel -def prepare_backlog(num_transactions=10000): - run('python3 benchmark_utils.py add-backlog {}'.format(num_transactions)) - - -@task -@parallel -def start_bigchaindb(): - run('screen -d -m bigchaindb start &', pty=False) - - -@task -@parallel -def kill_bigchaindb(): - run('killall bigchaindb') diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md deleted file mode 100644 index 38a4569b..00000000 --- a/benchmarking-tests/test1/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Transactions per second - -Measure how many blocks per second are created on the _bigchain_ with a pre filled backlog. - -1. Deploy an aws cluster https://docs.bigchaindb.com/projects/server/en/latest/clusters-feds/aws-testing-cluster.html -2. Make a symbolic link to hostlist.py: `ln -s ../deploy-cluster-aws/hostlist.py .` -3. Make a symbolic link to bigchaindb.pem: -```bash -mkdir pem -cd pem -ln -s ../deploy-cluster-aws/pem/bigchaindb.pem . -``` - -Then: - -```bash -fab put_benchmark_utils -fab prepare_backlog: # wait for process to finish -fab start_bigchaindb -``` diff --git a/codecov.yml b/codecov.yml index 547c6b99..0ab4582d 100644 --- a/codecov.yml +++ b/codecov.yml @@ -29,8 +29,6 @@ coverage: - "docs/*" - "tests/*" - "bigchaindb/version.py" - - "benchmarking-tests/*" - - "speed-tests/*" - "ntools/*" - "k8s/*" From 2560f02c36c08867c544858159cdb576160668ec Mon Sep 17 00:00:00 2001 From: Jack Riches Date: Mon, 3 Apr 2017 17:19:03 +0100 Subject: [PATCH 70/80] Fix syntax error (missing comma) --- bigchaindb/commands/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 15887340..9bec5a03 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -198,7 +198,7 @@ base_parser.add_argument('-c', '--config', '(use "-" for stdout)') base_parser.add_argument('-l', '--log-level', - type=str.upper # convert to uppercase for comparison to choices + type=str.upper, # convert to uppercase for comparison to choices choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='Log level') From d5c8d3067e06a95ae72d39e0bf9698bb6000ba68 Mon Sep 17 00:00:00 2001 From: Jack Riches Date: Mon, 3 Apr 2017 23:06:36 +0100 Subject: [PATCH 71/80] Use two spaces before inline comment (PEP8) (fix flake8 error) --- bigchaindb/commands/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 9bec5a03..f163a825 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -198,7 +198,7 @@ base_parser.add_argument('-c', '--config', '(use "-" for stdout)') base_parser.add_argument('-l', '--log-level', - type=str.upper, # convert to uppercase for comparison to choices + type=str.upper, # convert to uppercase for comparison to choices choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='Log level') From bb68a44b9674fd0a10f38254188b5526400eaba8 Mon Sep 17 00:00:00 2001 From: Anuj Date: Tue, 4 Apr 2017 13:55:24 +0530 Subject: [PATCH 72/80] Renamed bigchain import to bigchaindb --- tests/commands/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 2670725a..6fb424d6 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -152,7 +152,7 @@ def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): @patch('bigchaindb.backend.schema.drop_database') def test_drop_db_when_db_does_not_exist(mock_db_drop, capsys): from bigchaindb import config - from bigchaindb.commands.bigchain import run_drop + from bigchaindb.commands.bigchaindb import run_drop from bigchaindb.common.exceptions import DatabaseDoesNotExist args = Namespace(config=None, yes=True) mock_db_drop.side_effect = DatabaseDoesNotExist From 6f916d5781c3cea4e7c7d6447652953fe0b5be30 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 4 Apr 2017 11:25:26 +0200 Subject: [PATCH 73/80] Fixed documentation about transactions endpoint. (#1360) * Fixed documentation about transactions endpoint. * clarify how bigchaindb handles invalid transactions * rephrase --- .../source/drivers-clients/http-client-server-api.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 26ccd2f5..39e4395e 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -46,8 +46,12 @@ Transactions Get the transaction with the ID ``tx_id``. - This endpoint returns a transaction only if a ``VALID`` block on - ``bigchain`` exists. + This endpoint returns a transaction if it was included in a ``VALID`` block, + if it is still waiting to be processed (``BACKLOG``) or is still in an + undecided block (``UNDECIDED``). All instances of a transaction in invalid + blocks are ignored and treated as if they don't exist. If a request is made + for a transaction and instances of that transaction are found only in + invalid blocks, then the response will be ``404 Not Found``. :param tx_id: transaction ID :type tx_id: hex string From 1e07a5b111efff10451b6ca94f5dade14c8d0c58 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 14:51:02 +0200 Subject: [PATCH 74/80] Add ssl, login, and passwd to configure command --- bigchaindb/__init__.py | 39 +++++++++++++++++++++++----- bigchaindb/commands/bigchaindb.py | 17 +++++-------- bigchaindb/commands/utils.py | 42 +++++++++++++++++++++++++++++-- tests/commands/test_utils.py | 27 ++++++++++++++++++++ tests/test_config_utils.py | 12 +++++++++ 5 files changed, 118 insertions(+), 19 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index c0e4fd56..1be419b3 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,24 +5,51 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 -_database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + +_base_database_rethinkdb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'connection_timeout': 5000, - 'max_tries': 3, } -_database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), +# The following variable is used by `bigchaindb configure` to +# prompt the user for database values. We cannot rely on +# _base_database_rethinkdb.keys() or _base_database_mongodb.keys() +# because dicts are unordered. I tried to configure + +_base_database_rethinkdb_keys = ('host', 'port', 'name') + +_base_database_mongodb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), + 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), + 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), + 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') +} + +_base_database_mongodb_keys = ('host', 'port', 'name', 'replicaset', + 'ssl', 'login', 'password') + +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'connection_timeout': 5000, 'max_tries': 3, } +_database_rethinkdb.update(_base_database_rethinkdb) + +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'connection_timeout': 5000, + 'max_tries': 3, +} +_database_mongodb.update(_base_database_mongodb) + +_database_keys_map = { + 'mongodb': _base_database_mongodb_keys, + 'rethinkdb': _base_database_rethinkdb_keys +} _database_map = { 'mongodb': _database_mongodb, diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index ce0cbfa0..d4e37daa 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -88,26 +88,21 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend), file=sys.stderr) + database_keys = bigchaindb._database_keys_map[args.backend] conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: for key in ('bind', ): val = conf['server'][key] - conf['server'][key] = \ - input_on_stderr('API Server {}? (default `{}`): '.format(key, val)) \ - or val + conf['server'][key] = input_on_stderr('API Server {}? (default `{}`): '.format(key, val), val) - for key in ('host', 'port', 'name'): + for key in database_keys: val = conf['database'][key] - conf['database'][key] = \ - input_on_stderr('Database {}? (default `{}`): '.format(key, val)) \ - or val + conf['database'][key] = input_on_stderr('Database {}? (default `{}`): '.format(key, val), val) val = conf['backlog_reassign_delay'] - conf['backlog_reassign_delay'] = \ - input_on_stderr(('Stale transaction reassignment delay (in ' - 'seconds)? (default `{}`): '.format(val))) \ - or val + conf['backlog_reassign_delay'] = input_on_stderr( + 'Stale transaction reassignment delay (in seconds)? (default `{}`): '.format(val), val) if config_path != '-': bigchaindb.config_utils.write_config(conf, config_path) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index f163a825..cd59856c 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -74,12 +74,50 @@ def start_logging_process(command): return start_logging +def _convert(value, default=None, convert=None): + def convert_bool(value): + if value.lower() in ('true', 't', 'yes', 'y'): + return True + if value.lower() in ('false', 'f', 'no', 'n'): + return False + raise ValueError('{} cannot be converted to bool'.format(value)) + + if value == '': + value = None + + if convert is None: + if default is not None: + convert = type(default) + else: + convert = str + + if convert == bool: + convert = convert_bool + + if value is None: + return default + else: + return convert(value) + + # We need this because `input` always prints on stdout, while it should print # to stderr. It's a very old bug, check it out here: # - https://bugs.python.org/issue1927 -def input_on_stderr(prompt=''): +def input_on_stderr(prompt='', default=None, convert=None): + """Output a string to stderr and wait for input. + + Args: + prompt (str): the message to display. + default: the default value to return if the user + leaves the field empty + convert (callable): a callable to be used to convert + the value the user inserted. If None, the type of + ``default`` will be used. + """ + print(prompt, end='', file=sys.stderr) - return builtins.input() + value = builtins.input() + return _convert(value, default, convert) def start_rethinkdb(): diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index f3b64c18..85aa8de4 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -13,6 +13,33 @@ def reset_bigchaindb_config(monkeypatch): monkeypatch.setattr('bigchaindb.config', bigchaindb._config) +def test_input_on_stderr(): + from bigchaindb.commands.utils import input_on_stderr, _convert + + with patch('builtins.input', return_value='I love cats'): + assert input_on_stderr() == 'I love cats' + + # input_on_stderr uses `_convert` internally, from now on we will + # just use that function + + assert _convert('hack the planet') == 'hack the planet' + assert _convert('42') == '42' + assert _convert('42', default=10) == 42 + assert _convert('', default=10) == 10 + assert _convert('42', convert=int) == 42 + assert _convert('True', convert=bool) is True + assert _convert('False', convert=bool) is False + assert _convert('t', convert=bool) is True + assert _convert('3.14', default=1.0) == 3.14 + assert _convert('TrUe', default=False) is True + + with pytest.raises(ValueError): + assert _convert('TRVE', default=False) + + with pytest.raises(ValueError): + assert _convert('ಠ_ಠ', convert=int) + + @pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config') def test_configure_bigchaindb_configures_bigchaindb(): from bigchaindb.commands.utils import configure_bigchaindb diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 4234e242..51e4d595 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -19,6 +19,15 @@ def clean_config(monkeypatch, request): monkeypatch.setattr('bigchaindb.config', original_config) +def test_ordered_keys_match_database_config(): + import bigchaindb + + assert set(bigchaindb._base_database_rethinkdb.keys()) ==\ + set(bigchaindb._base_database_rethinkdb_keys) + assert set(bigchaindb._base_database_mongodb.keys()) ==\ + set(bigchaindb._base_database_mongodb_keys) + + def test_bigchain_instance_is_initialized_when_conf_provided(request): import bigchaindb from bigchaindb import config_utils @@ -181,6 +190,9 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'connection_timeout': 5000, 'max_tries': 3, 'replicaset': 'bigchain-rs', + 'ssl': False, + 'login': None, + 'password': None } database = {} From cb87221bdf2761f4d399b4da5cf45d8040f0587d Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 4 Apr 2017 17:57:44 +0200 Subject: [PATCH 75/80] Voting pipeline now checks for duplicated transactions in blocks during validation. --- bigchaindb/models.py | 9 +++++---- tests/pipelines/test_vote.py | 12 ++++++++++++ tests/test_models.py | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c371e792..2f46ba20 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -187,6 +187,11 @@ class Block(object): if not self.is_signature_valid(): raise InvalidSignature('Invalid block signature') + # Check that the block contains no duplicated transactions + txids = [tx.id for tx in self.transactions] + if len(txids) != len(set(txids)): + raise DuplicateTransaction('Block has duplicate transaction') + def _validate_block_transactions(self, bigchain): """Validate Block transactions. @@ -196,10 +201,6 @@ class Block(object): Raises: ValidationError: If an invalid transaction is found """ - txids = [tx.id for tx in self.transactions] - if len(txids) != len(set(txids)): - raise DuplicateTransaction('Block has duplicate transaction') - for tx in self.transactions: # If a transaction is not valid, `validate_transactions` will # throw an an exception and block validation will be canceled. diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index fa167d17..7df7ca11 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -111,6 +111,18 @@ def test_validate_block_with_invalid_id(b): assert invalid_dummy_tx == [vote_obj.invalid_dummy_tx] +@pytest.mark.genesis +def test_validate_block_with_duplicated_transactions(b): + from bigchaindb.pipelines import vote + + tx = dummy_tx(b) + block = b.create_block([tx, tx]).to_dict() + + vote_obj = vote.Vote() + block_id, invalid_dummy_tx = vote_obj.validate_block(block) + assert invalid_dummy_tx == [vote_obj.invalid_dummy_tx] + + @pytest.mark.genesis def test_validate_block_with_invalid_signature(b): from bigchaindb.pipelines import vote diff --git a/tests/test_models.py b/tests/test_models.py index db6a6975..6e559cb2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -152,4 +152,4 @@ class TestBlockModel(object): tx = Transaction.create([b.me], [([b.me], 1)]) block = b.create_block([tx, tx]) with raises(DuplicateTransaction): - block._validate_block_transactions(b) + block._validate_block(b) From 5d2f66524c04be4ef30f73732d5fb13ddab8ecae Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 4 Apr 2017 18:58:34 +0200 Subject: [PATCH 76/80] Cleanup configuration keys for db --- bigchaindb/__init__.py | 13 ++++--------- tests/test_config_utils.py | 10 ---------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 1be419b3..4c555e47 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -17,7 +17,10 @@ _base_database_rethinkdb = { # _base_database_rethinkdb.keys() or _base_database_mongodb.keys() # because dicts are unordered. I tried to configure -_base_database_rethinkdb_keys = ('host', 'port', 'name') +_database_keys_map = { + 'mongodb': ('host', 'port', 'name', 'replicaset'), + 'rethinkdb': ('host', 'port', 'name') +} _base_database_mongodb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), @@ -29,9 +32,6 @@ _base_database_mongodb = { 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') } -_base_database_mongodb_keys = ('host', 'port', 'name', 'replicaset', - 'ssl', 'login', 'password') - _database_rethinkdb = { 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'connection_timeout': 5000, @@ -46,11 +46,6 @@ _database_mongodb = { } _database_mongodb.update(_base_database_mongodb) -_database_keys_map = { - 'mongodb': _base_database_mongodb_keys, - 'rethinkdb': _base_database_rethinkdb_keys -} - _database_map = { 'mongodb': _database_mongodb, 'rethinkdb': _database_rethinkdb diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 51e4d595..04c70325 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -11,7 +11,6 @@ ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config) @pytest.fixture(scope='function', autouse=True) def clean_config(monkeypatch, request): - import bigchaindb original_config = copy.deepcopy(ORIGINAL_CONFIG) backend = request.config.getoption('--database-backend') @@ -19,15 +18,6 @@ def clean_config(monkeypatch, request): monkeypatch.setattr('bigchaindb.config', original_config) -def test_ordered_keys_match_database_config(): - import bigchaindb - - assert set(bigchaindb._base_database_rethinkdb.keys()) ==\ - set(bigchaindb._base_database_rethinkdb_keys) - assert set(bigchaindb._base_database_mongodb.keys()) ==\ - set(bigchaindb._base_database_mongodb_keys) - - def test_bigchain_instance_is_initialized_when_conf_provided(request): import bigchaindb from bigchaindb import config_utils From 09a440ee91b7bc3bd64069f70177338887981d04 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 5 Apr 2017 16:52:56 +0200 Subject: [PATCH 77/80] Fix get_spent incorrectly raising CriticalDoubleSpent --- bigchaindb/core.py | 55 +++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index e6783a6d..91f19f66 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -324,8 +324,12 @@ class Bigchain(object): def get_spent(self, txid, output): """Check if a `txid` was already used as an input. - A transaction can be used as an input for another transaction. Bigchain needs to make sure that a - given `txid` is only used once. + A transaction can be used as an input for another transaction. Bigchain + needs to make sure that a given `txid` is only used once. + + This method will check if the `txid` and `output` has already been + spent in a transaction that is in either the `VALID`, `UNDECIDED` or + `BACKLOG` state. Args: txid (str): The id of the transaction @@ -334,32 +338,43 @@ class Bigchain(object): Returns: The transaction (Transaction) that used the `txid` as an input else `None` + + Raises: + CriticalDoubleSpend: If the given `txid` and `output` was spent in + more than one valid transaction. """ # checks if an input was already spent # checks if the bigchain has any transaction with input {'txid': ..., # 'output': ...} - transactions = list(backend.query.get_spent(self.connection, txid, output)) + transactions = list(backend.query.get_spent(self.connection, txid, + output)) # a transaction_id should have been spent at most one time - if transactions: - # determine if these valid transactions appear in more than one valid block - num_valid_transactions = 0 - for transaction in transactions: - # ignore invalid blocks - # FIXME: Isn't there a faster solution than doing I/O again? - if self.get_transaction(transaction['id']): - num_valid_transactions += 1 - if num_valid_transactions > 1: - raise core_exceptions.CriticalDoubleSpend( - '`{}` was spent more than once. There is a problem' - ' with the chain'.format(txid)) + # determine if these valid transactions appear in more than one valid + # block + num_valid_transactions = 0 + non_invalid_transactions = [] + for transaction in transactions: + # ignore transactions in invalid blocks + # FIXME: Isn't there a faster solution than doing I/O again? + _, status = self.get_transaction(transaction['id'], + include_status=True) + if status == self.TX_VALID: + num_valid_transactions += 1 + # `txid` can only have been spent in at most on valid block. + if num_valid_transactions > 1: + raise core_exceptions.CriticalDoubleSpend( + '`{}` was spent more than once. There is a problem' + ' with the chain'.format(txid)) + # if its not and invalid transaction + if status is not None: + non_invalid_transactions.append(transaction) - if num_valid_transactions: - return Transaction.from_dict(transactions[0]) - else: - # all queried transactions were invalid - return None + if non_invalid_transactions: + return Transaction.from_dict(non_invalid_transactions[0]) else: + # Either no transaction was returned spending the `txid` as + # input or the returned transactions are not valid. return None def get_outputs(self, owner): From de04dcda0c0a51bdf24d5f0c15772c624a5a558d Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 6 Apr 2017 16:07:35 +0200 Subject: [PATCH 78/80] Fixed docstring. Removed redundant `else` branch. --- bigchaindb/core.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 91f19f66..5d2e9c03 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -325,9 +325,9 @@ class Bigchain(object): """Check if a `txid` was already used as an input. A transaction can be used as an input for another transaction. Bigchain - needs to make sure that a given `txid` is only used once. + needs to make sure that a given `(txid, output)` is only used once. - This method will check if the `txid` and `output` has already been + This method will check if the `(txid, output)` has already been spent in a transaction that is in either the `VALID`, `UNDECIDED` or `BACKLOG` state. @@ -336,11 +336,11 @@ class Bigchain(object): output (num): the index of the output in the respective transaction Returns: - The transaction (Transaction) that used the `txid` as an input else - `None` + The transaction (Transaction) that used the `(txid, output)` as an + input else `None` Raises: - CriticalDoubleSpend: If the given `txid` and `output` was spent in + CriticalDoubleSpend: If the given `(txid, output)` was spent in more than one valid transaction. """ # checks if an input was already spent @@ -372,10 +372,9 @@ class Bigchain(object): if non_invalid_transactions: return Transaction.from_dict(non_invalid_transactions[0]) - else: - # Either no transaction was returned spending the `txid` as - # input or the returned transactions are not valid. - return None + + # Either no transaction was returned spending the `(txid, output)` as + # input or the returned transactions are not valid. def get_outputs(self, owner): """Retrieve a list of links to transaction outputs for a given public From cf006e34a5135ef433b204530abd1c5a1d605003 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 6 Apr 2017 15:58:41 +0200 Subject: [PATCH 79/80] Make the keyword argument a keyword-only argument As per PEP 3102. This helps making the code clearer. --- bigchaindb/web/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index b1525f9f..6604a177 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -22,7 +22,7 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication): - http://docs.gunicorn.org/en/latest/custom.html """ - def __init__(self, app, options=None): + def __init__(self, app, *, options=None): '''Initialize a new standalone application. Args: @@ -91,5 +91,5 @@ def create_server(settings): settings['logger_class'] = 'bigchaindb.log.loggers.HttpServerLogger' app = create_app(debug=settings.get('debug', False), threads=settings['threads']) - standalone = StandaloneApplication(app, settings) + standalone = StandaloneApplication(app, options=settings) return standalone From c64a35c362c2dd71fddbeb59c7fcfc24e88cf66b Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 6 Apr 2017 16:01:42 +0200 Subject: [PATCH 80/80] Use new super syntax as per PEP 3135 --- bigchaindb/web/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 6604a177..46495368 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -32,7 +32,7 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication): ''' self.options = options or {} self.application = app - super(StandaloneApplication, self).__init__() + super().__init__() def load_config(self): config = dict((key, value) for key, value in self.options.items()