From 696dbe7844df014e0d308da613534b4290cd7e20 Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Tue, 14 Mar 2017 14:23:30 +0100 Subject: [PATCH 01/32] SSL connection support --- bigchaindb/backend/connection.py | 6 ++++-- bigchaindb/backend/mongodb/connection.py | 14 +++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index c1f0a629..cf6bece7 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None): + connection_timeout=None, replicaset=None, ssl=False): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -50,6 +50,8 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, # to handle these these additional args. In case of RethinkDBConnection # it just does not do anything with it. replicaset = replicaset or bigchaindb.config['database'].get('replicaset') + ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ + else ssl try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -63,7 +65,7 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, logger.debug('Connection: {}'.format(Class)) return Class(host=host, port=port, dbname=dbname, max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset) + replicaset=replicaset, ssl=ssl) class Connection: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 8688e243..274d64c1 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, **kwargs): + def __init__(self, replicaset=None, ssl=False, **kwargs): """Create a new Connection instance. Args: @@ -28,6 +28,8 @@ class MongoDBConnection(Connection): super().__init__(**kwargs) self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] + self.ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ + else ssl @property def db(self): @@ -71,14 +73,15 @@ class MongoDBConnection(Connection): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout) + initialize_replica_set(self.host, self.port, self.connection_timeout, self.ssl) # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. return pymongo.MongoClient(self.host, self.port, replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout) + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) # `initialize_replica_set` might raise `ConnectionFailure` or `OperationFailure`. except (pymongo.errors.ConnectionFailure, @@ -86,7 +89,7 @@ class MongoDBConnection(Connection): raise ConnectionError() from exc -def initialize_replica_set(host, port, connection_timeout): +def initialize_replica_set(host, port, connection_timeout, ssl): """Initialize a replica set. If already initialized skip.""" # Setup a MongoDB connection @@ -95,7 +98,8 @@ def initialize_replica_set(host, port, connection_timeout): # you try to connect to a replica set that is not yet initialized conn = pymongo.MongoClient(host=host, port=port, - serverselectiontimeoutms=connection_timeout) + serverselectiontimeoutms=connection_timeout, + ssl=ssl) _check_replica_set(conn) host = '{}:{}'.format(bigchaindb.config['database']['host'], bigchaindb.config['database']['port']) From 08f040d2186335e04560f97cc3ad844e74254319 Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 09:09:06 +0100 Subject: [PATCH 02/32] Authentication support --- bigchaindb/backend/connection.py | 6 ++++-- bigchaindb/backend/mongodb/connection.py | 27 +++++++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index cf6bece7..56b5cd82 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None, ssl=False): + connection_timeout=None, replicaset=None, ssl=False, login=None, password=None): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -52,6 +52,8 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, replicaset = replicaset or bigchaindb.config['database'].get('replicaset') ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ else ssl + login = login or bigchaindb.config['database'].get('login') + password = password or bigchaindb.config['database'].get('password') try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -65,7 +67,7 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, logger.debug('Connection: {}'.format(Class)) return Class(host=host, port=port, dbname=dbname, max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset, ssl=ssl) + replicaset=replicaset, ssl=ssl, login=login, password=password) class Connection: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 274d64c1..9168190a 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, ssl=False, **kwargs): + def __init__(self, replicaset=None, ssl=False, login=None, password=None, **kwargs): """Create a new Connection instance. Args: @@ -30,6 +30,8 @@ class MongoDBConnection(Connection): self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] self.ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ else ssl + self.login = login or bigchaindb.config['database'].get('login') + self.password = password or bigchaindb.config['database'].get('password') @property def db(self): @@ -73,15 +75,20 @@ class MongoDBConnection(Connection): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout, self.ssl) + initialize_replica_set(self.host, self.port, self.connection_timeout, self.dbname, self.ssl, self.login, self.password) # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. - return pymongo.MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout, - ssl=self.ssl) + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) + + if self.login is not None and self.password is not None: + client[self.dbname].authenticate(self.login, self.password) + + return client # `initialize_replica_set` might raise `ConnectionFailure` or `OperationFailure`. except (pymongo.errors.ConnectionFailure, @@ -89,7 +96,7 @@ class MongoDBConnection(Connection): raise ConnectionError() from exc -def initialize_replica_set(host, port, connection_timeout, ssl): +def initialize_replica_set(host, port, connection_timeout, dbname, ssl, login, password): """Initialize a replica set. If already initialized skip.""" # Setup a MongoDB connection @@ -100,6 +107,10 @@ def initialize_replica_set(host, port, connection_timeout, ssl): port=port, serverselectiontimeoutms=connection_timeout, ssl=ssl) + + if login is not None and password is not None: + conn[dbname].authenticate(login, password) + _check_replica_set(conn) host = '{}:{}'.format(bigchaindb.config['database']['host'], bigchaindb.config['database']['port']) From 8526246f7802141fc323540250672b9b248b88f6 Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 10:01:58 +0100 Subject: [PATCH 03/32] Fix unit test --- tests/backend/mongodb/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index 6350a7c5..a02b1735 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -168,7 +168,7 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] # check that it returns - assert initialize_replica_set('host', 1337, 1000) is None + assert initialize_replica_set('host', 1337, 1000, False, None, None) is None # test it raises OperationError if anything wrong with mock.patch.object(Database, 'command') as mock_command: @@ -178,4 +178,4 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] with pytest.raises(pymongo.errors.OperationFailure): - initialize_replica_set('host', 1337, 1000) + initialize_replica_set('host', 1337, 1000, False, None, None) From 3b1e6adb43b72d0b3ae1a13ffdaefcc6484f95cd Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 10:05:11 +0100 Subject: [PATCH 04/32] Formatting --- bigchaindb/backend/mongodb/connection.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 9168190a..8b30b2db 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -75,16 +75,17 @@ class MongoDBConnection(Connection): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout, self.dbname, self.ssl, self.login, self.password) + initialize_replica_set(self.host, self.port, self.connection_timeout, + self.dbname, self.ssl, self.login, self.password) # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. - client = pymongo.MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout, - ssl=self.ssl) - + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) + if self.login is not None and self.password is not None: client[self.dbname].authenticate(self.login, self.password) From 550b9cb804db8a6add84eea768b7b9355bbe27ac Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Fri, 17 Mar 2017 10:33:26 +0100 Subject: [PATCH 05/32] Fix unit test --- tests/backend/mongodb/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index a02b1735..e0b161b0 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -168,7 +168,7 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] # check that it returns - assert initialize_replica_set('host', 1337, 1000, False, None, None) is None + assert initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None) is None # test it raises OperationError if anything wrong with mock.patch.object(Database, 'command') as mock_command: @@ -178,4 +178,4 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] with pytest.raises(pymongo.errors.OperationFailure): - initialize_replica_set('host', 1337, 1000, False, None, None) + initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None) From e4ed122a1c56a25109aa323b166994921f2fb82d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 14:08:38 +0100 Subject: [PATCH 06/32] Correct default log datefmt for console --- bigchaindb/log/configs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bigchaindb/log/configs.py b/bigchaindb/log/configs.py index 7a8acc7c..1be5c485 100644 --- a/bigchaindb/log/configs.py +++ b/bigchaindb/log/configs.py @@ -18,9 +18,8 @@ SUBSCRIBER_LOGGING_CONFIG = { 'formatters': { 'console': { 'class': 'logging.Formatter', - 'format': ( - '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - ), + 'format': ('[%(asctime)s] [%(levelname)s] (%(name)s) ' + '%(message)s (%(processName)-10s - pid: %(process)d)'), 'datefmt': '%Y-%m-%d %H:%M:%S', }, 'file': { From 05db44a6366385f0210b869abc8cc05fb1550836 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 14:09:06 +0100 Subject: [PATCH 07/32] Add documentation for log configuration --- .../source/server-reference/configuration.md | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index f12b8247..2c94e870 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -22,6 +22,15 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_CONFIG_PATH`
`BIGCHAINDB_BACKLOG_REASSIGN_DELAY`
`BIGCHAINDB_CONSENSUS_PLUGIN`
+`BIGCHAINDB_LOG`
+`BIGCHAINDB_LOG_FILE`
+`BIGCHAINDB_LOG_LEVEL_CONSOLE`
+`BIGCHAINDB_LOG_LEVEL_LOGFILE`
+`BIGCHAINDB_LOG_DATEFMT_CONSOLE`
+`BIGCHAINDB_LOG_DATEFMT_LOGFILE`
+`BIGCHAINDB_LOG_FMT_CONSOLE`
+`BIGCHAINDB_LOG_FMT_LOGFILE`
+`BIGCHAINDB_LOG_GRANULAR_LEVELS`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -173,3 +182,209 @@ export BIGCHAINDB_CONSENSUS_PLUGIN=default ```js "consensus_plugin": "default" ``` + +## log +The `log` key is expected to point to a mapping (set of key/value pairs) +holding the logging configuration. + +**Example**: + +``` +{ + "log": { + "file": "/var/log/bigchaindb.log", + "level_console": "info", + "level_logfile": "info", + "datefmt_console": "%Y-%m-%d %H:%M:%S", + "datefmt_logfile": "%Y-%m-%d %H:%M:%S", + "fmt_console": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "fmt_logfile": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "granular_levels": { + "bichaindb.backend": "info", + "bichaindb.core": "info" + } +} +``` + +**Defaults to**: `"{}"`. + +Please note that although the default is `"{}"` as per the configuration file, +internal defaults are used, such that the actual operational default is: + +``` +{ + "log": { + "file": "~/bigchaindb.log", + "level_console": "info", + "level_logfile": "info", + "datefmt_console": "%Y-%m-%d %H:%M:%S", + "datefmt_logfile": "%Y-%m-%d %H:%M:%S", + "fmt_console": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "fmt_logfile": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "granular_levels": {} +} +``` + +The next subsections explain each field of the `log` configuration. + + +### log.file +The full path to the file where logs should be written to. + +**Example**: + +``` +{ + "log": { + "file": "/var/log/bigchaindb/bigchaindb.log" + } +} +``` + +**Defaults to**: `"~/bigchaindb.log"`. + +Please note that the user running `bigchaindb` must have write access to the +location. + + +### log.level_console +The log level used to log to the console. Possible allowed values are the ones +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): + +``` +"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +``` + +**Example**: + +``` +{ + "log": { + "level_console": "info" + } +} +``` + +**Defaults to**: `"info"`. + + +### log.level_logfile +The log level used to log to the log file. Possible allowed values are the ones +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): + +``` +"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +``` + +**Example**: + +``` +{ + "log": { + "level_file": "info" + } +} +``` + +**Defaults to**: `"info"`. + + +### log.datefmt_console +The format string for the date/time portion of a message, when logged to the +console. + +**Example**: + +``` +{ + "log": { + "datefmt_console": "%x %X %Z" + } +} +``` + +**Defaults to**: `"%Y-%m-%d %H:%M:%S"`. + +For more information on how to construct the format string please consult the +table under Python's documentation of + [`time.strftime(format[, t])`](https://docs.python.org/3.6/library/time.html#time.strftime) + +### log.datefmt_logfile +The format string for the date/time portion of a message, when logged to a log + file. + +**Example**: + +``` +{ + "log": { + "datefmt_logfile": "%c %z" + } +} +``` + +**Defaults to**: `"%Y-%m-%d %H:%M:%S"`. + +For more information on how to construct the format string please consult the +table under Python's documentation of + [`time.strftime(format[, t])`](https://docs.python.org/3.6/library/time.html#time.strftime) + + +### log.fmt_console +A string used to format the log messages when logged to the console. + +**Example**: + +``` +{ + "log": { + "fmt_console": "%(asctime)s [%(levelname)s] %(message)s %(process)d" + } +} +``` + +**Defaults to**: `"[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)"` + +For more information on possible formatting options please consult Python's +documentation on +[LogRecord attributes](https://docs.python.org/3.6/library/logging.html#logrecord-attributes) + + +### log.fmt_logfile +A string used to format the log messages when logged to a log file. + +**Example**: + +``` +{ + "log": { + "fmt_logfile": "%(asctime)s [%(levelname)s] %(message)s %(process)d" + } +} +``` + +**Defaults to**: `"[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)"` + +For more information on possible formatting options please consult Python's +documentation on +[LogRecord attributes](https://docs.python.org/3.6/library/logging.html#logrecord-attributes) + + +### log.granular_levels +Log levels for BigchainDB's modules. This can be useful to control the log +level of specific parts of the application. As an example, if you wanted the +`core.py` to be more verbose, you would set the configuration shown in the +example below. + +**Example**: + +``` +{ + "log": { + "granular_levels": { + "bichaindb.core": "debug" + } +} +``` + +**Defaults to**: `"{}"` From d867983a891fcb2c9f4fa1881da78bc581b37ba3 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 14:37:26 +0100 Subject: [PATCH 08/32] Document cmd line option for the log level --- docs/server/source/server-reference/bigchaindb-cli.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 9612fd30..88294621 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -68,6 +68,14 @@ You can also use the `--dev-start-rethinkdb` command line option to automaticall e.g. `bigchaindb --dev-start-rethinkdb start`. Note that this will also shutdown rethinkdb when the bigchaindb process stops. The option `--dev-allow-temp-keypair` will generate a keypair on the fly if no keypair is found, this is useful when you want to run a temporary instance of BigchainDB in a Docker container, for example. +### Options +The log level for the console can be set via the option `--log-level` or its +abbreviation `-l`. Example: + +```bash +$ bigchaindb --log-level INFO start +``` + ## bigchaindb set-shards From 433863798325beefd5ab8c1c77b055b4ec5641fd Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:43:00 +0100 Subject: [PATCH 09/32] Add docs for allowed log levels for cmd line --- docs/server/source/server-reference/bigchaindb-cli.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 88294621..31c955cd 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -76,6 +76,11 @@ abbreviation `-l`. Example: $ bigchaindb --log-level INFO start ``` +The allowed levels are `DEBUG`, `INFO` , `WARNING`, `ERROR`, and `CRITICAL`. +For an explanation regarding these levels please consult the +[Logging Levels](https://docs.python.org/3.6/library/logging.html#levels) +section of Python's documentation. + ## bigchaindb set-shards From 3a812701010c8d4e2f73ccadc17eba43763bbe80 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:54:18 +0100 Subject: [PATCH 10/32] Add link to configuration file settings --- docs/server/source/server-reference/bigchaindb-cli.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 31c955cd..05f321f9 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -81,6 +81,9 @@ For an explanation regarding these levels please consult the [Logging Levels](https://docs.python.org/3.6/library/logging.html#levels) section of Python's documentation. +For a more fine-grained control over the logging configuration you can use the +configuration file as documented under +[Configuration Settings](configuration.html). ## bigchaindb set-shards From 45ae58448f05793ce34517457db5fe7ffe045406 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:54:35 +0100 Subject: [PATCH 11/32] Correct wording --- docs/server/source/server-reference/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 2c94e870..32a6c3a0 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -373,8 +373,8 @@ documentation on ### log.granular_levels Log levels for BigchainDB's modules. This can be useful to control the log level of specific parts of the application. As an example, if you wanted the -`core.py` to be more verbose, you would set the configuration shown in the -example below. +logging of the `core.py` module to be more verbose, you would set the + configuration shown in the example below. **Example**: From 9987041ac04465ed4c53957ae8746ef38097cfba Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 21 Mar 2017 15:58:18 +0100 Subject: [PATCH 12/32] Add note about case insensitivity of log levels --- docs/server/source/server-reference/configuration.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 32a6c3a0..4cd9e9d4 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -249,10 +249,11 @@ location. ### log.level_console The log level used to log to the console. Possible allowed values are the ones -defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), +but case insensitive for convenience's sake: ``` -"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +"critical", "error", "warning", "info", "debug", "notset" ``` **Example**: @@ -270,10 +271,11 @@ defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): ### log.level_logfile The log level used to log to the log file. Possible allowed values are the ones -defined by [Python](https://docs.python.org/3.6/library/logging.html#levels): +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), +but case insensitive for convenience's sake: ``` -"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET" +"critical", "error", "warning", "info", "debug", "notset" ``` **Example**: From 10d83c2ab90822f3819eced18c701e0570e4cf84 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 22 Mar 2017 14:25:16 +0100 Subject: [PATCH 13/32] No duplicate vote inserts with mongodb (#1258) * prevent duplicate vote inserts --- bigchaindb/backend/mongodb/schema.py | 3 ++- tests/backend/mongodb/test_queries.py | 18 ++++++++++++++++++ tests/db/test_bigchain_api.py | 17 ----------------- tests/web/test_votes.py | 8 +++++++- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 4c5189ac..ad89f9bc 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -100,4 +100,5 @@ def create_votes_secondary_index(conn, dbname): ASCENDING), ('node_pubkey', ASCENDING)], - name='block_and_voter') + name='block_and_voter', + unique=True) diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index 1d7bfc39..bd7e75f1 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -212,6 +212,7 @@ def test_get_owned_ids(signed_create_tx, user_pk): def test_get_votes_by_block_id(signed_create_tx, structurally_valid_vote): + from bigchaindb.common.crypto import generate_key_pair from bigchaindb.backend import connect, query from bigchaindb.models import Block conn = connect() @@ -219,10 +220,14 @@ def test_get_votes_by_block_id(signed_create_tx, structurally_valid_vote): # create and insert a block block = Block(transactions=[signed_create_tx]) conn.db.bigchain.insert_one(block.to_dict()) + # create and insert some votes structurally_valid_vote['vote']['voting_for_block'] = block.id conn.db.votes.insert_one(structurally_valid_vote) + # create a second vote under a different key + _, pk = generate_key_pair() structurally_valid_vote['vote']['voting_for_block'] = block.id + structurally_valid_vote['node_pubkey'] = pk structurally_valid_vote.pop('_id') conn.db.votes.insert_one(structurally_valid_vote) @@ -325,6 +330,19 @@ def test_write_vote(structurally_valid_vote): assert vote_db == structurally_valid_vote +def test_duplicate_vote_raises_duplicate_key(structurally_valid_vote): + from bigchaindb.backend import connect, query + from bigchaindb.backend.exceptions import DuplicateKeyError + conn = connect() + + # write a vote + query.write_vote(conn, structurally_valid_vote) + + # write the same vote a second time + with pytest.raises(DuplicateKeyError): + query.write_vote(conn, structurally_valid_vote) + + def test_get_genesis_block(genesis_block): from bigchaindb.backend import connect, query conn = connect() diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 50d3f7b6..3f05385c 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -446,23 +446,6 @@ class TestBigchainApi(object): b.write_vote(b.vote(block_3.id, b.get_last_voted_block().id, True)) assert b.get_last_voted_block().id == block_3.id - def test_no_vote_written_if_block_already_has_vote(self, b, genesis_block): - from bigchaindb.models import Block - - block_1 = dummy_block() - b.write_block(block_1) - - b.write_vote(b.vote(block_1.id, genesis_block.id, True)) - retrieved_block_1 = b.get_block(block_1.id) - retrieved_block_1 = Block.from_dict(retrieved_block_1) - - # try to vote again on the retrieved block, should do nothing - b.write_vote(b.vote(retrieved_block_1.id, genesis_block.id, True)) - retrieved_block_2 = b.get_block(block_1.id) - retrieved_block_2 = Block.from_dict(retrieved_block_2) - - assert retrieved_block_1 == retrieved_block_2 - @pytest.mark.usefixtures('inputs') def test_assign_transaction_one_node(self, b, user_pk, user_sk): from bigchaindb.backend import query diff --git a/tests/web/test_votes.py b/tests/web/test_votes.py index bae31b9a..0bdd1081 100644 --- a/tests/web/test_votes.py +++ b/tests/web/test_votes.py @@ -27,6 +27,8 @@ def test_get_votes_endpoint(b, client): @pytest.mark.bdb @pytest.mark.usefixtures('inputs') def test_get_votes_endpoint_multiple_votes(b, client): + from bigchaindb.common.crypto import generate_key_pair + tx = Transaction.create([b.me], [([b.me], 1)]) tx = tx.sign([b.me_private]) @@ -37,8 +39,12 @@ def test_get_votes_endpoint_multiple_votes(b, client): vote_valid = b.vote(block.id, last_block, True) b.write_vote(vote_valid) - # vote the block valid + # vote the block invalid + # a note can only vote once so we need a new node_pubkey for the second + # vote + _, pk = generate_key_pair() vote_invalid = b.vote(block.id, last_block, False) + vote_invalid['node_pubkey'] = pk b.write_vote(vote_invalid) res = client.get(VOTES_ENDPOINT + '?block_id=' + block.id) From 425397f644f7f6427cd617dfc66a8386c5586ac1 Mon Sep 17 00:00:00 2001 From: Krish Date: Wed, 22 Mar 2017 14:25:25 +0100 Subject: [PATCH 14/32] NGINX frontend for MongoDB and BigchainDB (#1304) - Added NGINX deployment to frontend both BDB and MDB. - Nginx is configured with a whitelist (which is read from a ConfigMap) to allow only other MDB nodes in the closter to communicate with it. - Azure LB apparently does not support proxy protocol and hence whitelisting fails as nginx always observer the LB IP instead of the real IP in the TCP stream. - Whitelisting source IPs for MongoDB - Removing deprecated folder - Better log format - Intuitive port number usage - README and examples - Addressed a typo in PYTHON_STYLE_GUIDE.md - Azure LB apparently does not support proxy protocol and hence whitelisting fails as nginx always observer the LB IP instead of the real IP in the TCP stream. - Whitelisting source IPs for MongoDB - Removing deprecated folder - Multiple changes: - Better log format - Intuitive port number usage - README and examples - Addressed a typo in PYTHON_STYLE_GUIDE.md - Documentation - add the k8s directory to the ignore list in codecov.yml --- PYTHON_STYLE_GUIDE.md | 4 +- codecov.yml | 1 + .../add-node-on-kubernetes.rst | 15 ++ .../node-on-kubernetes.rst | 121 ++++++++++++---- .../template-kubernetes-azure.rst | 2 +- k8s/bigchaindb/bigchaindb-dep.yaml | 20 +-- k8s/deprecated.to.del/bdb-mdb-dep.yaml | 89 ------------ k8s/deprecated.to.del/bdb-rdb-dep.yaml | 87 ------------ k8s/deprecated.to.del/mongo-statefulset.yaml | 57 -------- k8s/deprecated.to.del/node-mdb-ss.yaml | 114 --------------- k8s/deprecated.to.del/node-rdb-ss.yaml | 131 ------------------ k8s/deprecated.to.del/node-ss.yaml | 89 ------------ k8s/deprecated.to.del/rethinkdb-ss.yaml | 75 ---------- k8s/mongodb/container/README.md | 2 +- k8s/mongodb/mongo-ss.yaml | 21 +-- k8s/nginx/container/Dockerfile | 11 ++ k8s/nginx/container/README.md | 70 ++++++++++ k8s/nginx/container/nginx.conf.template | 108 +++++++++++++++ k8s/nginx/container/nginx_entrypoint.bash | 44 ++++++ k8s/nginx/nginx-cm.yaml | 13 ++ k8s/nginx/nginx-dep.yaml | 82 +++++++++++ 21 files changed, 462 insertions(+), 694 deletions(-) delete mode 100644 k8s/deprecated.to.del/bdb-mdb-dep.yaml delete mode 100644 k8s/deprecated.to.del/bdb-rdb-dep.yaml delete mode 100644 k8s/deprecated.to.del/mongo-statefulset.yaml delete mode 100644 k8s/deprecated.to.del/node-mdb-ss.yaml delete mode 100644 k8s/deprecated.to.del/node-rdb-ss.yaml delete mode 100644 k8s/deprecated.to.del/node-ss.yaml delete mode 100644 k8s/deprecated.to.del/rethinkdb-ss.yaml create mode 100644 k8s/nginx/container/Dockerfile create mode 100644 k8s/nginx/container/README.md create mode 100644 k8s/nginx/container/nginx.conf.template create mode 100755 k8s/nginx/container/nginx_entrypoint.bash create mode 100644 k8s/nginx/nginx-cm.yaml create mode 100644 k8s/nginx/nginx-dep.yaml diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index befe4eeb..5ca44e83 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -82,6 +82,6 @@ flake8 --max-line-length 119 bigchaindb/ ## Writing and Running (Python) Tests -The content of this section was moved to [`bigchiandb/tests/README.md`](./tests/README.md). +The content of this section was moved to [`bigchaindb/tests/README.md`](./tests/README.md). -Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions. \ No newline at end of file +Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions. diff --git a/codecov.yml b/codecov.yml index b6f22af9..547c6b99 100644 --- a/codecov.yml +++ b/codecov.yml @@ -32,6 +32,7 @@ coverage: - "benchmarking-tests/*" - "speed-tests/*" - "ntools/*" + - "k8s/*" comment: # @stevepeak (from codecov.io) suggested we change 'suggestions' to 'uncovered' diff --git a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst index ea435ed3..7dcf1104 100644 --- a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst @@ -161,3 +161,18 @@ zero downtime during updates. You can SSH to an existing BigchainDB instance and run the ``bigchaindb show-config`` command to check that the keyring is updated. + + +Step 7: Run NGINX as a Deployment +--------------------------------- + +Please refer :ref:`this ` to +set up NGINX in your new node. + + +Step 8: Test Your New BigchainDB Node +------------------------------------- + +Please refer to the testing steps :ref:`here ` to verify that your new BigchainDB node is working as expected. + diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst index b19d79a3..6a59c750 100644 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -195,9 +195,9 @@ which can also be obtained using the ``az account list-locations`` command. You can also try to assign a name to an Public IP in Azure before starting the process, or use ``nslookup`` with the name you have in mind to check if it's available for use. -In the rare chance that name in the ``data.fqdn`` field is not available, -you must create a ConfigMap with a unique name and restart the -MongoDB instance. + +You should ensure that the the name specified in the ``data.fqdn`` field is +a unique one. **Kubernetes on bare-metal or other cloud providers.** You need to provide the name resolution function @@ -343,8 +343,8 @@ Get the file ``bigchaindb-dep.yaml`` from GitHub using: $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/bigchaindb/bigchaindb-dep.yaml -Note that we set the ``BIGCHAINDB_DATABASE_HOST`` to ``mdb`` which is the name -of the MongoDB service defined earlier. +Note that we set the ``BIGCHAINDB_DATABASE_HOST`` to ``mdb-svc`` which is the +name of the MongoDB service defined earlier. We also hardcode the ``BIGCHAINDB_KEYPAIR_PUBLIC``, ``BIGCHAINDB_KEYPAIR_PRIVATE`` and ``BIGCHAINDB_KEYRING`` for now. @@ -367,22 +367,55 @@ Create the required Deployment using: You can check its status using the command ``kubectl get deploy -w`` -Step 10: Verify the BigchainDB Node Setup +Step 10: Run NGINX as a Deployment +---------------------------------- + +NGINX is used as a proxy to both the BigchainDB and MongoDB instances in the +node. +It proxies HTTP requests on port 80 to the BigchainDB backend, and TCP +connections on port 27017 to the MongoDB backend. + +You can also configure a whitelist in NGINX to allow only connections from +other instances in the MongoDB replica set to access the backend MongoDB +instance. + +Get the file ``nginx-cm.yaml`` from GitHub using: + +.. code:: bash + + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/nginx/nginx-cm.yaml + +The IP address whitelist can be explicitly configured in ``nginx-cm.yaml`` +file. You will need a list of the IP addresses of all the other MongoDB +instances in the cluster. If the MongoDB intances specify a hostname, then this +needs to be resolved to the corresponding IP addresses. If the IP address of +any MongoDB instance changes, we can start a 'rolling upgrade' of NGINX after +updating the corresponding ConfigMap without affecting availabilty. + + +Create the ConfigMap for the whitelist using: + +.. code:: bash + + $ kubectl apply -f nginx-cm.yaml + +Get the file ``nginx-dep.yaml`` from GitHub using: + +.. code:: bash + + $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/nginx/nginx-dep.yaml + +Create the NGINX deployment using: + +.. code:: bash + + $ kubectl apply -f nginx-dep.yaml + + +Step 11: Verify the BigchainDB Node Setup ----------------------------------------- -Step 10.1: Testing Externally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Try to access the ``:9984`` -on your browser. You must receive a json output that shows the BigchainDB -server version among other things. - -Try to access the ``:27017`` -on your browser. You must receive a message from MongoDB stating that it -doesn't allow HTTP connections to the port anymore. - - -Step 10.2: Testing Internally +Step 11.1: Testing Internally ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Run a container that provides utilities like ``nslookup``, ``curl`` and ``dig`` @@ -392,23 +425,53 @@ on the cluster and query the internal DNS and IP endpoints. $ kubectl run -it toolbox -- image --restart=Never --rm -It will drop you to the shell prompt. -Now you can query for the ``mdb`` and ``bdb`` service details. - -.. code:: bash - - $ nslookup mdb - $ dig +noall +answer _mdb-port._tcp.mdb.default.svc.cluster.local SRV - $ curl -X GET http://mdb:27017 - $ curl -X GET http://bdb:9984 - There is a generic image based on alpine:3.5 with the required utilities hosted at Docker Hub under ``bigchaindb/toolbox``. The corresponding Dockerfile is `here `_. + You can use it as below to get started immediately: .. code:: bash $ kubectl run -it toolbox --image bigchaindb/toolbox --restart=Never --rm +It will drop you to the shell prompt. +Now you can query for the ``mdb`` and ``bdb`` service details. + +.. code:: bash + + # nslookup mdb-svc + # nslookup bdb-svc + # nslookup ngx-svc + # dig +noall +answer _mdb-port._tcp.mdb-svc.default.svc.cluster.local SRV + # dig +noall +answer _bdb-port._tcp.bdb-svc.default.svc.cluster.local SRV + # dig +noall +answer _ngx-public-mdb-port._tcp.ngx-svc.default.svc.cluster.local SRV + # dig +noall +answer _ngx-public-bdb-port._tcp.ngx-svc.default.svc.cluster.local SRV + # curl -X GET http://mdb-svc:27017 + # curl -X GET http://bdb-svc:9984 + # curl -X GET http://ngx-svc:80 + # curl -X GET http://ngx-svc:27017 + +The ``nslookup`` commands should output the configured IP addresses of the +services in the cluster + +The ``dig`` commands should return the port numbers configured for the +various services in the cluster. + +Finally, the ``curl`` commands test the availability of the services +themselves. + +Step 11.2: Testing Externally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Try to access the ``:80`` +on your browser. You must receive a json output that shows the BigchainDB +server version among other things. + +Try to access the ``:27017`` +on your browser. If your IP is in the whitelist, you will receive a message +from the MongoDB instance stating that it doesn't allow HTTP connections to +the port anymore. If your IP is not in the whitelist, your access will be +blocked and you will not see any response from the MongoDB instance. + diff --git a/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst b/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst index 93cf1e08..b967e764 100644 --- a/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst +++ b/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst @@ -168,7 +168,7 @@ using something like: .. code:: bash - $ ssh ssh ubuntu@k8s-agent-4AC80E97-0 + $ ssh ubuntu@k8s-agent-4AC80E97-0 where ``k8s-agent-4AC80E97-0`` is the name of a Kubernetes agent node in your Kubernetes cluster. diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 7bf68f06..83daaaaf 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -1,44 +1,47 @@ ############################################################### # This config file runs bigchaindb:master as a k8s Deployment # -# and it connects to the mongodb backend on a separate pod # +# and it connects to the mongodb backend running as a # +# separate pod # ############################################################### apiVersion: v1 kind: Service metadata: - name: bdb + name: bdb-svc namespace: default labels: - name: bdb + name: bdb-svc spec: selector: - app: bdb + app: bdb-dep ports: - port: 9984 targetPort: 9984 name: bdb-port - type: LoadBalancer + type: ClusterIP + clusterIP: None --- apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: bdb + name: bdb-dep spec: replicas: 1 template: metadata: labels: - app: bdb + app: bdb-dep spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb image: bigchaindb/bigchaindb:master + imagePullPolicy: IfNotPresent args: - start env: - name: BIGCHAINDB_DATABASE_HOST - value: mdb + value: mdb-svc - name: BIGCHAINDB_DATABASE_PORT # TODO(Krish): remove hardcoded port value: "27017" @@ -58,7 +61,6 @@ spec: value: "120" - name: BIGCHAINDB_KEYRING value: "" - imagePullPolicy: IfNotPresent ports: - containerPort: 9984 hostPort: 9984 diff --git a/k8s/deprecated.to.del/bdb-mdb-dep.yaml b/k8s/deprecated.to.del/bdb-mdb-dep.yaml deleted file mode 100644 index c985b285..00000000 --- a/k8s/deprecated.to.del/bdb-mdb-dep.yaml +++ /dev/null @@ -1,89 +0,0 @@ -############################################################### -# This config file runs bigchaindb:latest and connects to the # -# mongodb backend as a service # -############################################################### - -apiVersion: v1 -kind: Service -metadata: - name: bdb-mdb-service - namespace: default - labels: - name: bdb-mdb-service -spec: - selector: - app: bdb-mdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-api - type: LoadBalancer ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: bdb-mdb -spec: - replicas: 1 - template: - metadata: - labels: - app: bdb-mdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb-mdb - image: bigchaindb/bigchaindb:latest - args: - - start - env: - - name: BIGCHAINDB_DATABASE_HOST - value: mdb-service - - name: BIGCHAINDB_DATABASE_PORT - value: "27017" - - name: BIGCHAINDB_DATABASE_REPLICASET - value: bigchain-rs - - name: BIGCHIANDB_DATABASE_BACKEND - value: mongodb - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_SERVER_BIND - value: 0.0.0.0:9984 - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - - name: BIGCHAINDB_KEYRING - value: "" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - volumeMounts: - - name: bigchaindb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: bigchaindb-data - hostPath: - path: /disk/bigchaindb-data diff --git a/k8s/deprecated.to.del/bdb-rdb-dep.yaml b/k8s/deprecated.to.del/bdb-rdb-dep.yaml deleted file mode 100644 index 06daca43..00000000 --- a/k8s/deprecated.to.del/bdb-rdb-dep.yaml +++ /dev/null @@ -1,87 +0,0 @@ -############################################################### -# This config file runs bigchaindb:latest and connects to the # -# rethinkdb backend as a service # -############################################################### - -apiVersion: v1 -kind: Service -metadata: - name: bdb-rdb-service - namespace: default - labels: - name: bdb-rdb-service -spec: - selector: - app: bdb-rdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-rdb-api - type: LoadBalancer ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: bdb-rdb -spec: - replicas: 1 - template: - metadata: - labels: - app: bdb-rdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb-rdb - image: bigchaindb/bigchaindb:latest - args: - - start - env: - - name: BIGCHAINDB_DATABASE_HOST - value: rdb-service - - name: BIGCHAINDB_DATABASE_PORT - value: "28015" - - name: BIGCHIANDB_DATABASE_BACKEND - value: rethinkdb - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_SERVER_BIND - value: 0.0.0.0:9984 - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - - name: BIGCHAINDB_KEYRING - value: "" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - volumeMounts: - - name: bigchaindb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: bigchaindb-data - hostPath: - path: /disk/bigchaindb-data diff --git a/k8s/deprecated.to.del/mongo-statefulset.yaml b/k8s/deprecated.to.del/mongo-statefulset.yaml deleted file mode 100644 index a71567f3..00000000 --- a/k8s/deprecated.to.del/mongo-statefulset.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mongodb - labels: - name: mongodb -spec: - ports: - - port: 27017 - targetPort: 27017 - clusterIP: None - selector: - role: mongodb ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: mongodb -spec: - serviceName: mongodb - replicas: 3 - template: - metadata: - labels: - role: mongodb - environment: staging - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: mongo - image: mongo:3.4.1 - command: - - mongod - - "--replSet" - - bigchain-rs - #- "--smallfiles" - #- "--noprealloc" - ports: - - containerPort: 27017 - volumeMounts: - - name: mongo-persistent-storage - mountPath: /data/db - - name: mongo-sidecar - image: cvallance/mongo-k8s-sidecar - env: - - name: MONGO_SIDECAR_POD_LABELS - value: "role=mongo,environment=staging" - volumeClaimTemplates: - - metadata: - name: mongo-persistent-storage - annotations: - volume.beta.kubernetes.io/storage-class: "fast" - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 100Gi diff --git a/k8s/deprecated.to.del/node-mdb-ss.yaml b/k8s/deprecated.to.del/node-mdb-ss.yaml deleted file mode 100644 index 3c126d2d..00000000 --- a/k8s/deprecated.to.del/node-mdb-ss.yaml +++ /dev/null @@ -1,114 +0,0 @@ -################################################################# -# This YAML file desribes a StatefulSet with two containers: # -# bigchaindb/bigchaindb:latest and mongo:3.4.1 # -# It also describes a Service to expose BigchainDB and MongoDB. # -################################################################# - -apiVersion: v1 -kind: Service -metadata: - name: bdb-service - namespace: default - labels: - name: bdb-service -spec: - selector: - app: bdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-http-api - - port: 27017 - targetPort: 27017 - name: mongodb-port - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: bdb - namespace: default -spec: - serviceName: bdb - replicas: 1 - template: - metadata: - name: bdb - labels: - app: bdb - #annotations: - #pod.beta.kubernetes.io/init-containers: '[ - # TODO mongodb user and group; id = 999 - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bigchaindb - image: bigchaindb/bigchaindb:master - args: - - start - env: - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ - - name: BIGCHAINDB_KEYRING - value: "" - - name: BIGCHAINDB_DATABASE_BACKEND - value: mongodb - - name: BIGCHAINDB_DATABASE_HOST - value: localhost - - name: BIGCHAINDB_DATABASE_PORT - value: "27017" - - name: BIGCHAINDB_SERVER_BIND - value: "0.0.0.0:9984" - - name: BIGCHAINDB_DATABASE_REPLICASET - value: bigchain-rs - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: bdb-port - initialDelaySeconds: 15 - timeoutSeconds: 10 - - name: mongodb - image: mongo:3.4.1 - args: - - --replSet=bigchain-rs - imagePullPolicy: IfNotPresent - ports: - - containerPort: 27017 - hostPort: 27017 - name: mdb-port - protocol: TCP - volumeMounts: - - name: mdb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - tcpSocket: - port: mdb-port - successThreshold: 1 - failureThreshold: 3 - periodSeconds: 15 - timeoutSeconds: 1 - restartPolicy: Always - volumes: - - name: mdb-data - persistentVolumeClaim: - claimName: mongoclaim diff --git a/k8s/deprecated.to.del/node-rdb-ss.yaml b/k8s/deprecated.to.del/node-rdb-ss.yaml deleted file mode 100644 index fc157746..00000000 --- a/k8s/deprecated.to.del/node-rdb-ss.yaml +++ /dev/null @@ -1,131 +0,0 @@ -############################################################## -# This YAML file desribes a StatefulSet with two containers: # -# bigchaindb/bigchaindb:latest and rethinkdb:2.3 # -# It also describes a Service to expose BigchainDB, # -# the RethinkDB intracluster communications port, and # -# the RethinkDB web interface port. # -############################################################## - -apiVersion: v1 -kind: Service -metadata: - name: bdb-service - namespace: default - labels: - name: bdb-service -spec: - selector: - app: bdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-http-api - - port: 29015 - targetPort: 29015 - name: rdb-intracluster-comm-port - - port: 8080 - targetPort: 8080 - name: rdb-web-interface-port - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: bdb - namespace: default -spec: - serviceName: bdb - replicas: 1 - template: - metadata: - name: bdb - labels: - app: bdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb-server - image: bigchaindb/bigchaindb:latest - args: - - start - env: - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 56mEvwwVxcYsFQ3Y8UTFB8DVBv38yoUhxzDW3DAdLVd2 - - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: 9DsHwiEtvk51UHmNM2eV66czFha69j3CdtNrCj1RcZWR - - name: BIGCHAINDB_KEYRING - value: "" - - name: BIGCHAINDB_DATABASE_BACKEND - value: rethinkdb - - name: BIGCHAINDB_DATABASE_HOST - value: localhost - - name: BIGCHAINDB_DATABASE_PORT - value: "28015" - - name: BIGCHAINDB_SERVER_BIND - value: "0.0.0.0:9984" - - name: BIGCHAINDB_DATABASE_NAME - value: bigchain - - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY - value: "120" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - - name: rethinkdb - image: rethinkdb:2.3 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - hostPort: 8080 - name: rdb-web-interface-port - protocol: TCP - - containerPort: 29015 - hostPort: 29015 - name: rdb-intra-port - protocol: TCP - - containerPort: 28015 - hostPort: 28015 - name: rdb-client-port - protocol: TCP - volumeMounts: - - name: rdb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: rdb-data - persistentVolumeClaim: - claimName: mongoclaim diff --git a/k8s/deprecated.to.del/node-ss.yaml b/k8s/deprecated.to.del/node-ss.yaml deleted file mode 100644 index 9580daf6..00000000 --- a/k8s/deprecated.to.del/node-ss.yaml +++ /dev/null @@ -1,89 +0,0 @@ -##################################################### -# This config file uses bdb v0.9.1 with bundled rdb # -##################################################### - -apiVersion: v1 -kind: Service -metadata: - name: bdb-service - namespace: default - labels: - name: bdb-service -spec: - selector: - app: bdb - ports: - - port: 9984 - targetPort: 9984 - name: bdb-http-api - - port: 8080 - targetPort: 8080 - name: bdb-rethinkdb-api - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: bdb - namespace: default -spec: - serviceName: bdb - replicas: 1 - template: - metadata: - name: bdb - labels: - app: bdb - annotations: - pod.beta.kubernetes.io/init-containers: '[ - { - "name": "bdb091-configure", - "image": "bigchaindb/bigchaindb:0.9.1", - "command": ["bigchaindb", "-y", "configure", "rethinkdb"], - "volumeMounts": [ - { - "name": "bigchaindb-data", - "mountPath": "/data" - } - ] - } - ]' - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: bdb091-server - image: bigchaindb/bigchaindb:0.9.1 - args: - - -c - - /data/.bigchaindb - - start - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9984 - hostPort: 9984 - name: bdb-port - protocol: TCP - volumeMounts: - - name: bigchaindb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 9984 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: bigchaindb-data - hostPath: - path: /disk/bigchaindb-data diff --git a/k8s/deprecated.to.del/rethinkdb-ss.yaml b/k8s/deprecated.to.del/rethinkdb-ss.yaml deleted file mode 100644 index 081a5f6c..00000000 --- a/k8s/deprecated.to.del/rethinkdb-ss.yaml +++ /dev/null @@ -1,75 +0,0 @@ -#################################################### -# This config file runs rethinkdb:2.3 as a service # -#################################################### - -apiVersion: v1 -kind: Service -metadata: - name: rdb-service - namespace: default - labels: - name: rdb-service -spec: - selector: - app: rdb - ports: - - port: 8080 - targetPort: 8080 - name: rethinkdb-http-port - - port: 28015 - targetPort: 28015 - name: rethinkdb-driver-port - type: LoadBalancer ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: rdb - namespace: default -spec: - serviceName: rdb - replicas: 1 - template: - metadata: - name: rdb - labels: - app: rdb - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: rethinkdb - image: rethinkdb:2.3 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - hostPort: 8080 - name: rdb-http-port - protocol: TCP - - containerPort: 28015 - hostPort: 28015 - name: rdb-client-port - protocol: TCP - volumeMounts: - - name: rdb-data - mountPath: /data - resources: - limits: - cpu: 200m - memory: 768Mi - livenessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 15 - timeoutSeconds: 10 - restartPolicy: Always - volumes: - - name: rdb-data - hostPath: - path: /disk/rdb-data diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md index 7896a912..baad9f13 100644 --- a/k8s/mongodb/container/README.md +++ b/k8s/mongodb/container/README.md @@ -19,7 +19,7 @@ ``` docker run \ --name=mdb1 \ ---publish=17017:17017 \ +--publish=: \ --rm=true \ bigchaindb/mongodb \ --replica-set-name \ diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index fb6a73f8..089a0a96 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -1,38 +1,39 @@ ######################################################################## # This YAML file desribes a StatefulSet with a service for running and # -# exposing a MongoDB service. # +# exposing a MongoDB instance. # # It depends on the configdb and db k8s pvc. # ######################################################################## apiVersion: v1 kind: Service metadata: - name: mdb + name: mdb-svc namespace: default labels: - name: mdb + name: mdb-svc spec: selector: - app: mdb + app: mdb-ss ports: - port: 27017 targetPort: 27017 name: mdb-port - type: LoadBalancer + type: ClusterIP + clusterIP: None --- apiVersion: apps/v1beta1 kind: StatefulSet metadata: - name: mdb + name: mdb-ss namespace: default spec: - serviceName: mdb + serviceName: mdb-svc replicas: 1 template: metadata: - name: mdb + name: mdb-ss labels: - app: mdb + app: mdb-ss spec: terminationGracePeriodSeconds: 10 containers: @@ -41,6 +42,7 @@ spec: # versions during updates and rollbacks. Also, once fixed, change the # imagePullPolicy to IfNotPresent for faster bootup image: bigchaindb/mongodb:latest + imagePullPolicy: Always env: - name: MONGODB_FQDN valueFrom: @@ -60,7 +62,6 @@ spec: capabilities: add: - FOWNER - imagePullPolicy: Always ports: - containerPort: 27017 hostPort: 27017 diff --git a/k8s/nginx/container/Dockerfile b/k8s/nginx/container/Dockerfile new file mode 100644 index 00000000..c6c4dd3f --- /dev/null +++ b/k8s/nginx/container/Dockerfile @@ -0,0 +1,11 @@ +FROM nginx:1.11.10 +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /etc/nginx/nginx.conf +COPY nginx_entrypoint.bash / +EXPOSE 80 443 27017 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx/container/README.md b/k8s/nginx/container/README.md new file mode 100644 index 00000000..9cb44246 --- /dev/null +++ b/k8s/nginx/container/README.md @@ -0,0 +1,70 @@ +## Custom Nginx container for a Node + +### Need + +* Since, BigchainDB and MongoDB both need to expose ports to the outside + world (inter and intra cluster), we need to have a basic DDoS mitigation + strategy to ensure that we can provide proper uptime and security these + core services. + +* We can have a proxy like nginx/haproxy in every node that listens to + global connections and applies cluster level entry policy. + +### Implementation +* For MongoDB cluster communication, we will use nginx with an environment + variable specifying a ":" separated list of IPs in the whitelist. This list + contains the IPs of exising instances in the MongoDB replica set so as to + allow connections from the whitelist and avoid a DDoS. + +* For BigchainDB connections, nginx needs to have rules to throttle + connections that are using resources over a threshold. + + +### Step 1: Build the Latest Container + +Run `docker build -t bigchaindb/nginx .` from this folder. + +Optional: Upload container to Docker Hub: +`docker push bigchaindb/nginx:` + +### Step 2: Run the Container + +Note that the whilelist IPs must be specified with the subnet in the CIDR +format, eg: `1.2.3.4/16` + +``` +docker run \ +--env "MONGODB_FRONTEND_PORT=" \ +--env "MONGODB_BACKEND_HOST=" \ +--env "MONGODB_BACKEND_PORT=" \ +--env "BIGCHAINDB_FRONTEND_PORT=" \ +--env "BIGCHAINDB_BACKEND_HOST=" \ +--env "BIGCHAINDB_BACKEND_PORT=" \ +--env "MONGODB_WHITELIST=" \ +--name=ngx \ +--publish=: \ +--publish=: \ +--rm=true \ +bigchaindb/nginx +``` + +For example: +``` +docker run \ +--env "MONGODB_FRONTEND_PORT=17017" \ +--env "MONGODB_BACKEND_HOST=localhost" \ +--env "MONGODB_BACKEND_PORT=27017" \ +--env "BIGCHAINDB_FRONTEND_PORT=80" \ +--env "BIGCHAINDB_BACKEND_HOST=localhost" \ +--env "BIGCHAINDB_BACKEND_PORT=9984" \ +--env "MONGODB_WHITELIST="192.168.0.0/16:10.0.2.0/24" \ +--name=ngx \ +--publish=80:80 \ +--publish=17017:17017 \ +--rm=true \ +bigchaindb/nginx +``` + diff --git a/k8s/nginx/container/nginx.conf.template b/k8s/nginx/container/nginx.conf.template new file mode 100644 index 00000000..eda3e7c7 --- /dev/null +++ b/k8s/nginx/container/nginx.conf.template @@ -0,0 +1,108 @@ +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /etc/nginx/nginx.error.log; + +events { + worker_connections 256; + accept_mutex on; + use epoll; +} + +http { + server_names_hash_bucket_size 128; + resolver 8.8.8.8 8.8.4.4; + access_log /etc/nginx/nginx.access.log combined buffer=16k flush=5s; + + # allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # enable logging when requests are being throttled + limit_req_log_level notice; + + # the http status code to return to the client when throttling; + # 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + upstream bdb_backend { + server BIGCHAINDB_BACKEND_HOST:BIGCHAINDB_BACKEND_PORT max_fails=5 fail_timeout=30; + } + + server { + listen BIGCHAINDB_FRONTEND_PORT; + # server_name "FRONTEND_DNS_NAME"; + underscores_in_headers on; + + # max client request body size: avg transaction size + client_max_body_size 15k; + + # keepalive connection settings + keepalive_timeout 20s; + + # `slowloris` attack mitigation settings + client_body_timeout 10s; + client_header_timeout 10s; + + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_redirect off; + + # TODO proxy_set_header X-Forwarded-Proto https; + + # limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + proxy_pass http://bdb_backend; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /etc/nginx/50x.html; + } + } +} + +# NGINX stream block for TCP and UDP proxies +stream { + log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /etc/nginx/nginx.stream.access.log mdb_log buffer=16k flush=5s; + + # define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address + limit_conn_zone $binary_remote_addr zone=two:10m; + + # enable logging when connections are being throttled + limit_conn_log_level notice; + + upstream mdb_backend { + server MONGODB_BACKEND_HOST:MONGODB_BACKEND_PORT max_fails=5 fail_timeout=30 max_conns=1024; + } + + server { + listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5; + preread_timeout 30s; + tcp_nodelay on; + + # whitelist + MONGODB_WHITELIST + # deny access to everyone else + deny all; + + # allow 512 connections from the same IP address + limit_conn two 512; + + proxy_pass mdb_backend; + } +} diff --git a/k8s/nginx/container/nginx_entrypoint.bash b/k8s/nginx/container/nginx_entrypoint.bash new file mode 100755 index 00000000..9b63e278 --- /dev/null +++ b/k8s/nginx/container/nginx_entrypoint.bash @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail + +mongo_frontend_port=`printenv MONGODB_FRONTEND_PORT` +mongo_backend_host=`printenv MONGODB_BACKEND_HOST` +mongo_backend_port=`printenv MONGODB_BACKEND_PORT` +bdb_frontend_port=`printenv BIGCHAINDB_FRONTEND_PORT` +bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` +bdb_backend_port=`printenv BIGCHAINDB_BACKEND_PORT` +mongo_whitelist=`printenv MONGODB_WHITELIST` + +# sanity checks +if [[ -z "${mongo_frontend_port}" || \ + -z "${mongo_backend_host}" || \ + -z "${mongo_backend_port}" || \ + -z "${bdb_frontend_port}" || \ + -z "${bdb_backend_host}" || \ + -z "${bdb_backend_port}" ]] ; then + echo "Invalid environment settings detected. Exiting!" + exit 1 +fi + +NGINX_CONF_FILE=/etc/nginx/nginx.conf + +# configure the nginx.conf file with env variables +sed -i "s|MONGODB_FRONTEND_PORT|${mongo_frontend_port}|g" $NGINX_CONF_FILE +sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" $NGINX_CONF_FILE +sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" $NGINX_CONF_FILE +sed -i "s|BIGCHAINDB_FRONTEND_PORT|${bdb_frontend_port}|g" $NGINX_CONF_FILE +sed -i "s|BIGCHAINDB_BACKEND_HOST|${bdb_backend_host}|g" $NGINX_CONF_FILE +sed -i "s|BIGCHAINDB_BACKEND_PORT|${bdb_backend_port}|g" $NGINX_CONF_FILE + +# populate the whitelist in the conf file as per MONGODB_WHITELIST env var +hosts=$(echo ${mongo_whitelist} | tr ":" "\n") +for host in $hosts; do + sed -i "s|MONGODB_WHITELIST|allow ${host};\n MONGODB_WHITELIST|g" $NGINX_CONF_FILE +done + +# remove the MONGODB_WHITELIST marker string from template +sed -i "s|MONGODB_WHITELIST||g" $NGINX_CONF_FILE + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c /etc/nginx/nginx.conf diff --git a/k8s/nginx/nginx-cm.yaml b/k8s/nginx/nginx-cm.yaml new file mode 100644 index 00000000..7a255aae --- /dev/null +++ b/k8s/nginx/nginx-cm.yaml @@ -0,0 +1,13 @@ +######################################################################### +# This YAML file desribes a ConfigMap with a valid list of IP addresses # +# that can connect to the MongoDB instance. # +######################################################################### + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-whitelist + namespace: default +data: + # ':' separated list of allowed hosts + allowed-hosts: 192.168.0.0/16:10.0.2.0/24 diff --git a/k8s/nginx/nginx-dep.yaml b/k8s/nginx/nginx-dep.yaml new file mode 100644 index 00000000..d7739a56 --- /dev/null +++ b/k8s/nginx/nginx-dep.yaml @@ -0,0 +1,82 @@ +############################################################### +# This config file runs nginx as a k8s deployment and exposes # +# it using an external load balancer. # +# This deployment is used as a front end to both BigchainDB # +# and MongoDB. # +############################################################### + +apiVersion: v1 +kind: Service +metadata: + name: ngx-svc + namespace: default + labels: + name: ngx-svc + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-dep + ports: + - port: 27017 + targetPort: 27017 + name: ngx-public-mdb-port + protocol: TCP + - port: 80 + targetPort: 80 + name: ngx-public-bdb-port + protocol: TCP + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: ngx-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: bigchaindb/nginx:latest + imagePullPolicy: Always + env: + - name: MONGODB_FRONTEND_PORT + value: "27017" + - name: MONGODB_BACKEND_HOST + value: mdb-svc + - name: MONGODB_BACKEND_PORT + value: "27017" + - name: BIGCHAINDB_FRONTEND_PORT + value: "80" + - name: BIGCHAINDB_BACKEND_HOST + value: bdb-svc + - name: BIGCHAINDB_BACKEND_PORT + value: "9984" + - name: MONGODB_WHITELIST + valueFrom: + configMapKeyRef: + name: mongodb-whitelist + key: allowed-hosts + ports: + - containerPort: 27017 + hostPort: 27017 + name: public-mdb-port + protocol: TCP + - containerPort: 80 + hostPort: 80 + name: public-bdb-port + protocol: TCP + resources: + limits: + cpu: 200m + memory: 768Mi + #livenessProbe: TODO(Krish) + #readinessProbe: TODO(Krish) + restartPolicy: Always From f98a634d65e3be5a0aa5f06e2fea13843fc73c70 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 22 Mar 2017 14:37:37 +0100 Subject: [PATCH 15/32] clarify allowed maximum complexity of conditions --- docs/server/source/data-models/inputs-outputs.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index 9f1b5d56..e81aa3b2 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -22,7 +22,12 @@ One can also put different weights on the inputs to a threshold condition, along The (single) output of a threshold condition can be used as one of the inputs of other threshold conditions. This means that one can combine threshold conditions to build complex logical expressions, e.g. (x OR y) AND (u OR v). -When one creates a condition, one can calculate its fulfillment length (e.g. 96). The more complex the condition, the larger its fulfillment length will be. A BigchainDB federation can put an upper limit on the allowed fulfillment length, as a way of capping the complexity of conditions (and the computing time required to validate them). +When one creates a condition, one can calculate its fulfillment length (e.g. +96). The more complex the condition, the larger its fulfillment length will be. +A BigchainDB federation can put an upper limit on the complexity of the +conditions, either directly by setting an allowed maximum fulfillment length, +or indirectly by setting a maximum allowed transaction size which would limit +the overall complexity accross all inputs and outputs of a transaction. If someone tries to make a condition where the output of a threshold condition feeds into the input of another “earlier” threshold condition (i.e. in a closed logical circuit), then their computer will take forever to calculate the (infinite) “condition URI”, at least in theory. In practice, their computer will run out of memory or their client software will timeout after a while. From 0ae9d19a542a6c9882eac864ad827cf903e4fbf8 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 14:33:25 +0100 Subject: [PATCH 16/32] Separate log configuration from logging process Closes #1317 --- bigchaindb/commands/bigchain.py | 4 +- bigchaindb/commands/utils.py | 50 +++++++++++++++++++---- tests/commands/rethinkdb/test_commands.py | 16 ++------ tests/commands/test_commands.py | 42 ++++--------------- tests/commands/test_utils.py | 19 ++++----- 5 files changed, 66 insertions(+), 65 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 767f6ccc..be17d75f 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -24,7 +24,8 @@ from bigchaindb.commands.messages import ( CANNOT_START_KEYPAIR_NOT_FOUND, RETHINKDB_STARTUP_ERROR, ) -from bigchaindb.commands.utils import configure_bigchaindb, input_on_stderr +from bigchaindb.commands.utils import ( + configure_bigchaindb, start_logging_process, input_on_stderr) logging.basicConfig(level=logging.INFO) @@ -169,6 +170,7 @@ def run_drop(args): @configure_bigchaindb +@start_logging_process def run_start(args): """Start the processes to run the node""" logger.info('BigchainDB Version %s', bigchaindb.__version__) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 73313f05..f4a311fa 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -21,20 +21,56 @@ from bigchaindb.version import __version__ def configure_bigchaindb(command): + """Decorator to be used by command line functions, such that the + configuration of bigchaindb is performed before the execution of + the command. + + Args: + command: The command to decorate. + + Returns: + The command wrapper function. + + """ @functools.wraps(command) def configure(args): - bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) - - logging_config = bigchaindb.config['log'] or {} - if 'log_level' in args and args.log_level: - logging_config['level_console'] = args.log_level - setup_logging(user_log_config=logging_config) - + try: + config_from_cmdline = {'log': {'level_console': args.log_level}} + except AttributeError: + config_from_cmdline = None + bigchaindb.config_utils.autoconfigure( + filename=args.config, config=config_from_cmdline, force=True) command(args) return configure +def start_logging_process(command): + """Decorator to start the logging subscriber process. + + Args: + command: The command to decorate. + + Returns: + The command wrapper function. + + .. important:: + + Configuration, if needed, should be applied before invoking this + decorator, as starting the subscriber process for logging will + configure the root logger for the child process based on the + state of :obj:`bigchaindb.config` at the moment this decorator + is invoked. + + """ + @functools.wraps(command) + def start_logging(args): + from bigchaindb import config + setup_logging(user_log_config=config.get('log')) + command(args) + return start_logging + + # We need this because `input` always prints on stdout, while it should print # to stderr. It's a very old bug, check it out here: # - https://bugs.python.org/issue1927 diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index ac100075..165fef0d 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -38,7 +38,7 @@ def test_start_rethinkdb_exits_when_cannot_start(mock_popen): @patch('rethinkdb.ast.Table.reconfigure') -def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): +def test_set_shards(mock_reconfigure, monkeypatch, b): from bigchaindb.commands.bigchain import run_set_shards # this will mock the call to retrieve the database config @@ -50,8 +50,6 @@ def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): args = Namespace(num_shards=3, config=None) run_set_shards(args) mock_reconfigure.assert_called_with(replicas=1, shards=3, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() # this will mock the call to retrieve the database config # we will set it to return three replica @@ -61,10 +59,9 @@ def test_set_shards(mock_reconfigure, monkeypatch, b, mocked_setup_logging): monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_replicas) run_set_shards(args) mock_reconfigure.assert_called_with(replicas=3, shards=3, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_set_shards_raises_exception(monkeypatch, b, mocked_setup_logging): +def test_set_shards_raises_exception(monkeypatch, b): from bigchaindb.commands.bigchain import run_set_shards # test that we are correctly catching the exception @@ -81,11 +78,10 @@ def test_set_shards_raises_exception(monkeypatch, b, mocked_setup_logging): with pytest.raises(SystemExit) as exc: run_set_shards(args) assert exc.value.args == ('Failed to reconfigure tables.',) - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('rethinkdb.ast.Table.reconfigure') -def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): +def test_set_replicas(mock_reconfigure, monkeypatch, b): from bigchaindb.commands.bigchain import run_set_replicas # this will mock the call to retrieve the database config @@ -97,8 +93,6 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): args = Namespace(num_replicas=2, config=None) run_set_replicas(args) mock_reconfigure.assert_called_with(replicas=2, shards=2, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() # this will mock the call to retrieve the database config # we will set it to return three shards @@ -108,10 +102,9 @@ def test_set_replicas(mock_reconfigure, monkeypatch, b, mocked_setup_logging): monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_shards) run_set_replicas(args) mock_reconfigure.assert_called_with(replicas=2, shards=3, dry_run=False) - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_set_replicas_raises_exception(monkeypatch, b, mocked_setup_logging): +def test_set_replicas_raises_exception(monkeypatch, b): from bigchaindb.commands.bigchain import run_set_replicas # test that we are correctly catching the exception @@ -128,4 +121,3 @@ def test_set_replicas_raises_exception(monkeypatch, b, mocked_setup_logging): with pytest.raises(SystemExit) as exc: run_set_replicas(args) assert exc.value.args == ('Failed to reconfigure tables.',) - mocked_setup_logging.assert_called_once_with(user_log_config={}) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index eebd86ea..50b995b0 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -74,7 +74,7 @@ def test_bigchain_run_start_assume_yes_create_default_config( # interfere with capsys. # See related issue: https://github.com/pytest-dev/pytest/issues/128 @pytest.mark.usefixtures('ignore_local_config_file') -def test_bigchain_show_config(capsys, mocked_setup_logging): +def test_bigchain_show_config(capsys): from bigchaindb import config from bigchaindb.commands.bigchain import run_show_config @@ -85,11 +85,9 @@ def test_bigchain_show_config(capsys, mocked_setup_logging): del config['CONFIGURED'] config['keypair']['private'] = 'x' * 45 assert output_config == config - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch, - mocked_setup_logging): +def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): from bigchaindb import config from bigchaindb.commands.bigchain import run_export_my_pubkey @@ -106,11 +104,9 @@ def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch, lines = out.splitlines() assert config['keypair']['public'] in lines assert 'Charlie_Bucket' in lines - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch, - mocked_setup_logging): +def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): from bigchaindb import config from bigchaindb.commands.bigchain import run_export_my_pubkey @@ -126,49 +122,41 @@ def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch, # https://docs.python.org/3/library/exceptions.html#SystemExit assert exc_info.value.code == \ "This node's public key wasn't set anywhere so it can't be exported" - mocked_setup_logging.assert_called_once_with(user_log_config={}) -def test_bigchain_run_init_when_db_exists(mocked_setup_logging, - mock_db_init_with_existing_db): +def test_bigchain_run_init_when_db_exists(mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_init args = Namespace(config=None) run_init(args) - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_when_assumed_yes(mock_db_drop, mocked_setup_logging): +def test_drop_db_when_assumed_yes(mock_db_drop): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=True) run_drop(args) assert mock_db_drop.called - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch, - mocked_setup_logging): +def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=False) monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'y') run_drop(args) assert mock_db_drop.called - mocked_setup_logging.assert_called_once_with(user_log_config={}) @patch('bigchaindb.backend.schema.drop_database') -def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch, - mocked_setup_logging): +def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=False) monkeypatch.setattr('bigchaindb.commands.bigchain.input_on_stderr', lambda x: 'n') run_drop(args) assert not mock_db_drop.called - mocked_setup_logging.assert_called_once_with(user_log_config={}) def test_run_configure_when_config_exists_and_skipping(monkeypatch): @@ -417,7 +405,7 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, @pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.commands.bigchain.add_replicas') -def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): +def test_run_add_replicas(mock_add_replicas): from bigchaindb.commands.bigchain import run_add_replicas from bigchaindb.backend.exceptions import OperationError @@ -427,9 +415,7 @@ def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): mock_add_replicas.return_value = None assert run_add_replicas(args) is None assert mock_add_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() - mocked_setup_logging.reset_mock() # test add_replicas with `OperationError` mock_add_replicas.side_effect = OperationError('err') @@ -437,9 +423,7 @@ def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): run_add_replicas(args) assert exc.value.args == ('err',) assert mock_add_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() - mocked_setup_logging.reset_mock() # test add_replicas with `NotImplementedError` mock_add_replicas.side_effect = NotImplementedError('err') @@ -447,14 +431,12 @@ def test_run_add_replicas(mock_add_replicas, mocked_setup_logging): run_add_replicas(args) assert exc.value.args == ('err',) assert mock_add_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) mock_add_replicas.reset_mock() - mocked_setup_logging.reset_mock() @pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.commands.bigchain.remove_replicas') -def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): +def test_run_remove_replicas(mock_remove_replicas): from bigchaindb.commands.bigchain import run_remove_replicas from bigchaindb.backend.exceptions import OperationError @@ -464,8 +446,6 @@ def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): mock_remove_replicas.return_value = None assert run_remove_replicas(args) is None assert mock_remove_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() # test add_replicas with `OperationError` @@ -474,8 +454,6 @@ def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): run_remove_replicas(args) assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() # test add_replicas with `NotImplementedError` @@ -484,6 +462,4 @@ def test_run_remove_replicas(mock_remove_replicas, mocked_setup_logging): run_remove_replicas(args) assert exc.value.args == ('err',) assert mock_remove_replicas.call_count == 1 - mocked_setup_logging.assert_called_once_with(user_log_config={}) - mocked_setup_logging.reset_mock() mock_remove_replicas.reset_mock() diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py index 223c1f99..5f190717 100644 --- a/tests/commands/test_utils.py +++ b/tests/commands/test_utils.py @@ -1,7 +1,6 @@ import argparse from argparse import ArgumentTypeError, Namespace import logging -from logging import getLogger import pytest @@ -15,7 +14,7 @@ def reset_bigchaindb_config(monkeypatch): @pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config') -def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): +def test_configure_bigchaindb_configures_bigchaindb(): from bigchaindb.commands.utils import configure_bigchaindb from bigchaindb.config_utils import is_configured assert not is_configured() @@ -26,7 +25,6 @@ def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): args = Namespace(config=None) test_configure(args) - mocked_setup_logging.assert_called_once_with(user_log_config={}) @pytest.mark.usefixtures('ignore_local_config_file', @@ -39,22 +37,19 @@ def test_configure_bigchaindb_configures_bigchaindb(mocked_setup_logging): logging.ERROR, logging.CRITICAL, )) -def test_configure_bigchaindb_configures_logging(log_level, - mocked_setup_sub_logger): +def test_configure_bigchaindb_logging(log_level): from bigchaindb.commands.utils import configure_bigchaindb - from bigchaindb.log.configs import PUBLISHER_LOGGING_CONFIG - root_logger = getLogger() - assert root_logger.level == logging.NOTSET + from bigchaindb import config + assert not config['log'] @configure_bigchaindb def test_configure_logger(args): - root_logger = getLogger() - assert root_logger.level == PUBLISHER_LOGGING_CONFIG['root']['level'] + pass args = Namespace(config=None, log_level=log_level) test_configure_logger(args) - mocked_setup_sub_logger.assert_called_once_with( - user_log_config={'level_console': log_level}) + from bigchaindb import config + assert config['log'] == {'level_console': log_level} def test_start_raises_if_command_not_implemented(): From b42264e27e934d34e0bbc63bf988ee1cb5120629 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 14:46:21 +0100 Subject: [PATCH 17/32] Add commands subpackage documentation --- docs/server/source/appendices/commands.rst | 18 ++++++++++++++++++ docs/server/source/appendices/index.rst | 1 + 2 files changed, 19 insertions(+) create mode 100644 docs/server/source/appendices/commands.rst diff --git a/docs/server/source/appendices/commands.rst b/docs/server/source/appendices/commands.rst new file mode 100644 index 00000000..35d37b27 --- /dev/null +++ b/docs/server/source/appendices/commands.rst @@ -0,0 +1,18 @@ +###################### +Command Line Interface +###################### + +.. automodule:: bigchaindb.commands + :special-members: __init__ + + +:mod:`bigchaindb.commands.bigchain` +----------------------------------- + +.. automodule:: bigchaindb.commands.bigchain + + +:mod:`bigchaindb.commands.utils` +-------------------------------- + +.. automodule:: bigchaindb.commands.utils diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 365bedfa..7beb27f5 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -16,6 +16,7 @@ Appendices consensus pipelines backend + commands aws-setup generate-key-pair-for-ssh firewall-notes From 054fb48ca8f4be1b88e864cd979b19edc58f75a4 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 16:36:10 +0100 Subject: [PATCH 18/32] Set cmd line log level option default --- bigchaindb/commands/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index f4a311fa..b67c8ee8 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -196,6 +196,7 @@ base_parser.add_argument('-c', '--config', base_parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Log level') base_parser.add_argument('-y', '--yes', '--yes-please', From 6e3f25a1432eaa1ed9cc7a7e30a115130d5af25a Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 22 Mar 2017 17:39:04 +0100 Subject: [PATCH 19/32] Set log level for gunicorn --- bigchaindb/commands/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index b67c8ee8..cf8ddb4f 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -35,7 +35,10 @@ def configure_bigchaindb(command): @functools.wraps(command) def configure(args): try: - config_from_cmdline = {'log': {'level_console': args.log_level}} + config_from_cmdline = { + 'log': {'level_console': args.log_level}, + 'server': {'loglevel': args.log_level}, + } except AttributeError: config_from_cmdline = None bigchaindb.config_utils.autoconfigure( From cea78b3ae2aa3db943027788c3dd2e32e1f2490a Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 23 Mar 2017 15:28:01 +0100 Subject: [PATCH 20/32] Integrate gunicorn logs with bigchaindb logs Closes #1329 --- bigchaindb/__init__.py | 1 + bigchaindb/log/configs.py | 5 +++++ bigchaindb/log/loggers.py | 32 ++++++++++++++++++++++++++++++++ bigchaindb/log/setup.py | 9 +++++++-- tests/log/test_loggers.py | 18 ++++++++++++++++++ tests/test_config_utils.py | 1 + 6 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 bigchaindb/log/loggers.py create mode 100644 tests/log/test_loggers.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index c0e4fd56..00085314 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -36,6 +36,7 @@ config = { 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 + 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': _database_map[ os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') diff --git a/bigchaindb/log/configs.py b/bigchaindb/log/configs.py index 1be5c485..9dac0dcb 100644 --- a/bigchaindb/log/configs.py +++ b/bigchaindb/log/configs.py @@ -1,7 +1,12 @@ import logging +from logging.handlers import DEFAULT_TCP_LOGGING_PORT from os.path import expanduser, join +DEFAULT_SOCKET_LOGGING_HOST = 'localhost' +DEFAULT_SOCKET_LOGGING_PORT = DEFAULT_TCP_LOGGING_PORT +DEFAULT_SOCKET_LOGGING_ADDR = (DEFAULT_SOCKET_LOGGING_HOST, + DEFAULT_SOCKET_LOGGING_PORT) DEFAULT_LOG_DIR = expanduser('~') PUBLISHER_LOGGING_CONFIG = { diff --git a/bigchaindb/log/loggers.py b/bigchaindb/log/loggers.py new file mode 100644 index 00000000..f8c18320 --- /dev/null +++ b/bigchaindb/log/loggers.py @@ -0,0 +1,32 @@ +import logging.handlers + +from gunicorn.glogging import Logger + +from .configs import DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT + + +class HttpServerLogger(Logger): + """Custom logger class for ``gunicorn`` logs. + + Meant for internal usage only, to set the ``logger_class`` + configuration setting on gunicorn. + + """ + def setup(self, cfg): + """Setup the gunicorn access and error loggers. This overrides + the parent method. Its main goal is to simply pipe all the logs to + the TCP socket used througout BigchainDB. + + Args: + cfg (:obj:`gunicorn.config.Config`): Gunicorn configuration + object. *Ignored*. + + """ + self._set_socklog_handler(self.error_log) + self._set_socklog_handler(self.access_log) + + def _set_socklog_handler(self, log): + socket_handler = logging.handlers.SocketHandler( + DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT) + socket_handler._gunicorn = True + log.addHandler(socket_handler) diff --git a/bigchaindb/log/setup.py b/bigchaindb/log/setup.py index fdf8e49b..f3e8f7a3 100644 --- a/bigchaindb/log/setup.py +++ b/bigchaindb/log/setup.py @@ -9,7 +9,12 @@ import struct import sys from multiprocessing import Process -from .configs import PUBLISHER_LOGGING_CONFIG, SUBSCRIBER_LOGGING_CONFIG +from .configs import ( + DEFAULT_SOCKET_LOGGING_HOST, + DEFAULT_SOCKET_LOGGING_PORT, + PUBLISHER_LOGGING_CONFIG, + SUBSCRIBER_LOGGING_CONFIG, +) from bigchaindb.common.exceptions import ConfigurationError @@ -23,7 +28,7 @@ def _normalize_log_level(level): def setup_pub_logger(): dictConfig(PUBLISHER_LOGGING_CONFIG) socket_handler = logging.handlers.SocketHandler( - 'localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) + DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT) socket_handler.setLevel(logging.DEBUG) logger = logging.getLogger() logger.addHandler(socket_handler) diff --git a/tests/log/test_loggers.py b/tests/log/test_loggers.py new file mode 100644 index 00000000..795de046 --- /dev/null +++ b/tests/log/test_loggers.py @@ -0,0 +1,18 @@ +from logging.handlers import SocketHandler + + +class TestHttpServerLogger: + + def test_init(self, mocker): + from bigchaindb.log.configs import ( + DEFAULT_SOCKET_LOGGING_ADDR as expected_socket_address) + from bigchaindb.log.loggers import HttpServerLogger + mocked_config = mocker.patch( + 'gunicorn.config.Config', autospec=True, spec_set=True) + logger = HttpServerLogger(mocked_config.return_value) + assert len(logger.access_log.handlers) == 1 + assert len(logger.error_log.handlers) == 1 + assert isinstance(logger.access_log.handlers[0], SocketHandler) + assert isinstance(logger.error_log.handlers[0], SocketHandler) + assert logger.access_log.handlers[0].address == expected_socket_address + assert logger.error_log.handlers[0].address == expected_socket_address diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 4234e242..2e843914 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -195,6 +195,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'bind': SERVER_BIND, 'workers': None, 'threads': None, + 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': database, 'keypair': { From f5a32e35c5f2bdea89fcf79cb821e6f694b050f7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 23 Mar 2017 18:21:55 +0100 Subject: [PATCH 21/32] docs: 1st draft of page about updating all s/w on a BDB node on k8s --- .../cloud-deployment-templates/index.rst | 2 + .../upgrade-on-kubernetes.rst | 105 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 837dc66d..28ac7923 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -16,3 +16,5 @@ If you find the cloud deployment templates for nodes helpful, then you may also template-kubernetes-azure node-on-kubernetes add-node-on-kubernetes + upgrade-on-kubernetes + \ No newline at end of file diff --git a/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst new file mode 100644 index 00000000..348abf22 --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst @@ -0,0 +1,105 @@ +Kubernetes Template: Upgrade all Software in a BigchainDB Node +============================================================== + +This page outlines how to upgrade all the software associated +with a BigchainDB node running on Kubernetes, +including host operating systems, Docker, Kubernetes, +and BigchainDB-related software. + + +Upgrade Host OS, Docker and Kubernetes +-------------------------------------- + +Some Kubernetes installation & management systems +can do full or partial upgrades of host OSes, Docker, +or Kubernetes, e.g. +`Tectonic `_, +`Rancher `_, +and +`Kubo `_. +Consult the documentation for your system. + +**Azure Container Service (ACS).** +On Dec. 15, 2016, a Microsoft employee +`wrote `_: +"In the coming months we [the Azure Kubernetes team] will be building managed updates in the ACS service." +At the time of writing, managed updates were not yet available, +but you should check the latest +`ACS documentation `_ +to see what's available now. +Also at the time of writing, ACS only supported Ubuntu +as the host (master and agent) operating system. +You can upgrade Ubuntu and Docker on Azure +by SSHing into each of the hosts, +as documented on +:ref:`another page `. + +In general, you can SSH to each host in your Kubernetes Cluster +to update the OS and Docker. + +.. note:: + + Once you are in an SSH session with a host, + the ``docker info`` command is a handy way to detemine the + host OS (including version) and the Docker version. + +When you want to upgrade the software on a Kubernetes node, +you should "drain" the node first, +i.e. tell Kubernetes to gracefully terminate all pods +on the node and mark it as unscheduleable +(so no new pods get put on the node during its downtime). + +.. code:: + + kubectl drain $NODENAME + +There are `more details in the Kubernetes docs `_, +including instructions to make the node scheduleable again. + +To manually upgrade the host OS, +see the docs for that OS. + +To manually upgrade Docker, see +`the Docker docs `_. + +To manually upgrade all Kubernetes software in your Kubernetes cluster, see +`the Kubernetes docs `_. + + +Upgrade BigchainDB-Related Software +----------------------------------- + +We use Kubernetes "Deployments" for NGINX, BigchainDB, +and most other BigchainDB-related software. +The only exception is MongoDB; we use a Kubernetes +StatefulSet for that. + +The nice thing about Kubernetes Deployments +is that Kubernetes can manage most of the upgrade process. +A typical upgrade workflow for a single Deployment would be: + +.. code:: + + $ KUBE_EDITOR=nano kubectl edit deployment/ + +The `kubectl edit `_ +command opens the specified editor (nano in the above example), +allowing you to edit the specified Deployment *in the Kubernetes cluster*. +You can change the version tag on the Docker image, for example. +Don't forget to save your edits before exiting the editor. +The Kubernetes docs have more information about +`updating a Deployment `_. + + +The upgrade story for the MongoDB StatefulSet is *different*. +(This is because MongoDB has persistent state, +which is stored in some storage associated with a PersistentVolumeClaim.) +At the time of writing, StatefulSets were still in beta, +and they did not support automated image upgrade (Docker image tag upgrade). +We expect that to change. +Rather than trying to keep these docs up-to-date, +we advise you to check out the current +`Kubernetes docs about updating containers in StatefulSets +`_. + + From c61e3333d0e37405dcb2a7d2f30e46672bb44fa0 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 24 Mar 2017 10:57:56 +0100 Subject: [PATCH 22/32] improved readthedocs.org instructions in Release_Process.md --- Release_Process.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index ec51ceaf..22572837 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -45,10 +45,16 @@ These steps are common between minor and patch releases: 1. Make sure your local Git is in the same state as the release: e.g. `git fetch ` and `git checkout v0.9.1` 1. Make sure you have a `~/.pypirc` file containing credentials for PyPI 1. Do a `make release` to build and publish the new `bigchaindb` package on PyPI -1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. - Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is - "Active" and "Public", and make sure the new version's branch - (without the 'v' in front) is _not_ active -1. Also in readthedocs.org, go to Admin --> Advanced Settings - and make sure that "Default branch:" (i.e. what "latest" points to) - is set to the new release's tag, e.g. `v0.9.1`. (Don't miss the 'v' in front.) +1. [Login to readthedocs.org](https://readthedocs.org/accounts/login/) + as a maintainer of the BigchainDB Server docs, and: + - Go to Admin --> Advanced Settings + and make sure that "Default branch:" (i.e. what "latest" points to) + is set to the new release's tag, e.g. `v0.9.1`. + (Don't miss the 'v' in front.) + - Go to Admin --> Versions + and under **Choose Active Versions**, do these things: + 1. Make sure that the new version's tag is "Active" and "Public" + 2. Make sure the new version's branch + (without the 'v' in front) is _not_ active. + 3. Make sure the **stable** branch is _not_ active. + 4. Scroll to the bottom of the page and click the Submit button. From 0edb1c18f2c5f678cba86ad5c91a30502e2c7c6d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 24 Mar 2017 11:56:35 +0100 Subject: [PATCH 23/32] Keep gunicorn logger_class internal closes #1334 --- bigchaindb/__init__.py | 1 - bigchaindb/web/server.py | 1 + tests/test_config_utils.py | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 00085314..c0e4fd56 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -36,7 +36,6 @@ config = { 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 - 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': _database_map[ os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index bcd44d11..b1525f9f 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -88,6 +88,7 @@ def create_server(settings): if not settings.get('threads'): settings['threads'] = (multiprocessing.cpu_count() * 2) + 1 + settings['logger_class'] = 'bigchaindb.log.loggers.HttpServerLogger' app = create_app(debug=settings.get('debug', False), threads=settings['threads']) standalone = StandaloneApplication(app, settings) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 2e843914..4234e242 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -195,7 +195,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'bind': SERVER_BIND, 'workers': None, 'threads': None, - 'logger_class': 'bigchaindb.log.loggers.HttpServerLogger', }, 'database': database, 'keypair': { From 58d80e9731333eced2bbd513cf9f71f4281fc6bf Mon Sep 17 00:00:00 2001 From: Thomas Conte Date: Mon, 27 Mar 2017 10:43:40 +0200 Subject: [PATCH 24/32] Fix ssl param default value --- bigchaindb/backend/connection.py | 5 ++--- bigchaindb/backend/mongodb/connection.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 56b5cd82..b717703b 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None, ssl=False, login=None, password=None): + connection_timeout=None, replicaset=None, ssl=None, login=None, password=None): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -50,8 +50,7 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, # to handle these these additional args. In case of RethinkDBConnection # it just does not do anything with it. replicaset = replicaset or bigchaindb.config['database'].get('replicaset') - ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ - else ssl + ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False) login = login or bigchaindb.config['database'].get('login') password = password or bigchaindb.config['database'].get('password') diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 8b30b2db..5c54470a 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, ssl=False, login=None, password=None, **kwargs): + def __init__(self, replicaset=None, ssl=None, login=None, password=None, **kwargs): """Create a new Connection instance. Args: @@ -28,8 +28,7 @@ class MongoDBConnection(Connection): super().__init__(**kwargs) self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] - self.ssl = bigchaindb.config['database'].get('ssl') if bigchaindb.config['database'].get('ssl') is not None \ - else ssl + self.ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False) self.login = login or bigchaindb.config['database'].get('login') self.password = password or bigchaindb.config['database'].get('password') From 441ad914cf854ba64769ed3ce1bf5dd911fc9e24 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 11:24:16 +0200 Subject: [PATCH 25/32] Improve test coverage --- tests/backend/mongodb/test_connection.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index e0b161b0..3edc31b1 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -99,6 +99,18 @@ def test_connection_run_errors(mock_client, mock_init_repl_set): assert query.run.call_count == 1 +@mock.patch('pymongo.database.Database.authenticate') +def test_connection_with_credentials(mock_authenticate): + import bigchaindb + from bigchaindb.backend.mongodb.connection import MongoDBConnection + conn = MongoDBConnection(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port'], + login='theplague', + password='secret') + conn.connect() + assert mock_authenticate.call_count == 2 + + def test_check_replica_set_not_enabled(mongodb_connection): from bigchaindb.backend.mongodb.connection import _check_replica_set from bigchaindb.common.exceptions import ConfigurationError From 9679561d89c0c0fcf22c67dfb9b9a83737c5543d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 28 Mar 2017 12:14:50 +0200 Subject: [PATCH 26/32] Update pytest command in Makefile Use new command "pytest", add verbosity, and distribute tests across available CPU cores --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7fc9c1c0..afa992bf 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,7 @@ lint: ## check style with flake8 flake8 bigchaindb tests test: ## run tests quickly with the default Python - py.test - + pytest -v -n auto test-all: ## run tests on every Python version with tox tox From 1083e04dd5eaad5289470fe478a37e03f32ec3ee Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 28 Mar 2017 12:17:35 +0200 Subject: [PATCH 27/32] Fix Makefile (test) coverage target --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index afa992bf..37bf6db8 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,8 @@ test-all: ## run tests on every Python version with tox tox coverage: ## check code coverage quickly with the default Python - coverage run --source bigchaindb py.test - - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html + pytest -v -n auto --cov=bigchaindb --cov-report term --cov-report html + $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs $(MAKE) -C docs/root clean From 699e615d47bb6b24f4d373c8edfd3a84d4b6e0c1 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 14:51:02 +0200 Subject: [PATCH 28/32] Add ssl, login, and passwd to configure command --- bigchaindb/__init__.py | 36 +++++++++++++++++++++++++++------ bigchaindb/commands/bigchain.py | 3 ++- tests/test_config_utils.py | 12 +++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 1df2551c..53e7fd2b 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,24 +5,48 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 -_database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + +_base_database_rethinkdb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'connection_timeout': 5000, - 'max_tries': 3, } -_database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), +# This might sound excessive, but having an order on the keys will +# stress users (and us) less. +_base_database_rethinkdb_keys = ('host', 'port', 'name') + +_base_database_mongodb = { 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), + 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), + 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), + 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') +} + +_base_database_mongodb_keys = ('host', 'port', 'name', 'replicaset', + 'ssl', 'login', 'password') + +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'connection_timeout': 5000, 'max_tries': 3, } +_database_rethinkdb.update(_base_database_rethinkdb) + +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'connection_timeout': 5000, + 'max_tries': 3, +} +_database_mongodb.update(_base_database_mongodb) + +_database_keys_map = { + 'mongodb': _base_database_mongodb_keys, + 'rethinkdb': _base_database_rethinkdb_keys +} _database_map = { 'mongodb': _database_mongodb, diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index efefa9d7..a5ec9c6a 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -90,6 +90,7 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend), file=sys.stderr) + database_keys = bigchaindb._database_keys_map[args.backend] conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: @@ -99,7 +100,7 @@ def run_configure(args, skip_if_exists=False): input_on_stderr('API Server {}? (default `{}`): '.format(key, val)) \ or val - for key in ('host', 'port', 'name'): + for key in database_keys: val = conf['database'][key] conf['database'][key] = \ input_on_stderr('Database {}? (default `{}`): '.format(key, val)) \ diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 0fa5135b..d81f5f75 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -19,6 +19,15 @@ def clean_config(monkeypatch, request): monkeypatch.setattr('bigchaindb.config', original_config) +def test_ordered_keys_match_database_config(): + import bigchaindb + + assert set(bigchaindb._base_database_rethinkdb.keys()) ==\ + set(bigchaindb._base_database_rethinkdb_keys) + assert set(bigchaindb._base_database_mongodb.keys()) ==\ + set(bigchaindb._base_database_mongodb_keys) + + def test_bigchain_instance_is_initialized_when_conf_provided(request): import bigchaindb from bigchaindb import config_utils @@ -181,6 +190,9 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'connection_timeout': 5000, 'max_tries': 3, 'replicaset': 'bigchain-rs', + 'ssl': False, + 'login': None, + 'password': None } database = {} From 047108046acd57dcd6bbddd344d5a213def1d985 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 28 Mar 2017 15:01:10 +0200 Subject: [PATCH 29/32] Revert "Add ssl, login, and passwd to configure command" This reverts commit 699e615d47bb6b24f4d373c8edfd3a84d4b6e0c1. --- bigchaindb/__init__.py | 36 ++++++--------------------------- bigchaindb/commands/bigchain.py | 3 +-- tests/test_config_utils.py | 12 ----------- 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 53e7fd2b..1df2551c 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,48 +5,24 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 - -_base_database_rethinkdb = { +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), + 'connection_timeout': 5000, + 'max_tries': 3, } -# This might sound excessive, but having an order on the keys will -# stress users (and us) less. -_base_database_rethinkdb_keys = ('host', 'port', 'name') - -_base_database_mongodb = { +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), - 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), - 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') -} - -_base_database_mongodb_keys = ('host', 'port', 'name', 'replicaset', - 'ssl', 'login', 'password') - -_database_rethinkdb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'connection_timeout': 5000, 'max_tries': 3, } -_database_rethinkdb.update(_base_database_rethinkdb) - -_database_mongodb = { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), - 'connection_timeout': 5000, - 'max_tries': 3, -} -_database_mongodb.update(_base_database_mongodb) - -_database_keys_map = { - 'mongodb': _base_database_mongodb_keys, - 'rethinkdb': _base_database_rethinkdb_keys -} _database_map = { 'mongodb': _database_mongodb, diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index a5ec9c6a..efefa9d7 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -90,7 +90,6 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend), file=sys.stderr) - database_keys = bigchaindb._database_keys_map[args.backend] conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: @@ -100,7 +99,7 @@ def run_configure(args, skip_if_exists=False): input_on_stderr('API Server {}? (default `{}`): '.format(key, val)) \ or val - for key in database_keys: + for key in ('host', 'port', 'name'): val = conf['database'][key] conf['database'][key] = \ input_on_stderr('Database {}? (default `{}`): '.format(key, val)) \ diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index d81f5f75..0fa5135b 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -19,15 +19,6 @@ def clean_config(monkeypatch, request): monkeypatch.setattr('bigchaindb.config', original_config) -def test_ordered_keys_match_database_config(): - import bigchaindb - - assert set(bigchaindb._base_database_rethinkdb.keys()) ==\ - set(bigchaindb._base_database_rethinkdb_keys) - assert set(bigchaindb._base_database_mongodb.keys()) ==\ - set(bigchaindb._base_database_mongodb_keys) - - def test_bigchain_instance_is_initialized_when_conf_provided(request): import bigchaindb from bigchaindb import config_utils @@ -190,9 +181,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'connection_timeout': 5000, 'max_tries': 3, 'replicaset': 'bigchain-rs', - 'ssl': False, - 'login': None, - 'password': None } database = {} From c6de90fa79c080bb1403260b36f873e0ed1b4a69 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 29 Mar 2017 09:56:41 +0200 Subject: [PATCH 30/32] Upgrade rapidjson to latest 0.0.11 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a38bb1f..5fb201f4 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ install_requires = [ 'pymongo~=3.4', 'pysha3~=1.0.2', 'cryptoconditions>=0.5.0', - 'python-rapidjson==0.0.8', + 'python-rapidjson==0.0.11', 'logstats>=0.2.1', 'flask>=0.10.1', 'flask-restful~=0.3.0', From ead832a130ed2136dac7735e502af5b708468b3a Mon Sep 17 00:00:00 2001 From: morrme Date: Mon, 27 Mar 2017 05:13:29 -0500 Subject: [PATCH 31/32] added Python 3.6 per issue #1331 --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index da7ae05f..e558d154 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ cache: pip python: - 3.4 - 3.5 - + - 3.6 + env: - TOXENV=flake8 - TOXENV=docsroot @@ -30,6 +31,12 @@ matrix: env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb - python: 3.5 env: BIGCHAINDB_DATABASE_BACKEND=mongodb + - python: 3.6 + addons: + rethinkdb: '2.3.5' + env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb + - python: 3.6 + env: BIGCHAINDB_DATABASE_BACKEND=mongodb before_install: sudo .ci/travis-before-install.sh From 5c2bab078fac8bdd65a3ec77a8e524084fff0c70 Mon Sep 17 00:00:00 2001 From: morrme Date: Mon, 27 Mar 2017 05:14:40 -0500 Subject: [PATCH 32/32] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d2cd2a2c..8f299471 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = true -envlist = py{34,35}-{rethinkdb,mongodb}, flake8, docsroot, docsserver +envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver [base] basepython = python3.5