From e41ac6c6344c96ce199e234e5eaca865ba36c95d Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 16 Jun 2017 13:22:01 +0200 Subject: [PATCH 01/42] Added docs about the wsserver.scheme config setting --- docs/server/source/server-reference/configuration.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 04fed917..7eeb6508 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -182,12 +182,14 @@ export BIGCHAINDB_SERVER_THREADS=5 ``` -## wsserver.host and wsserver.port +## wsserver.scheme, wsserver.host and wsserver.port These settings are for the [aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), which is used to serve the [WebSocket Event Stream API](../websocket-event-stream-api.html). +`wsserver.scheme` should be either `"ws"` or `"wss"` +(but setting it to `"wss"` does *not* enable SSL/TLS). `wsserver.host` is where to bind the aiohttp server socket and `wsserver.port` is the corresponding port. If you want to allow connections from anyone, on port 9985, @@ -195,6 +197,7 @@ set `wsserver.host` to 0.0.0.0 and `wsserver.port` to 9985. **Example using environment variables** ```text +export BIGCHAINDB_WSSERVER_SCHEME=ws export BIGCHAINDB_WSSERVER_HOST=0.0.0.0 export BIGCHAINDB_WSSERVER_PORT=9985 ``` @@ -202,6 +205,7 @@ export BIGCHAINDB_WSSERVER_PORT=9985 **Example config file snippet** ```js "wsserver": { + "scheme": "wss", "host": "0.0.0.0", "port": 65000 } @@ -210,6 +214,7 @@ export BIGCHAINDB_WSSERVER_PORT=9985 **Default values (from a config file)** ```js "wsserver": { + "scheme": "ws", "host": "localhost", "port": 9985 } From 0e8d6944a8b450fbb00f097b5cfc8a6679e9785c Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 10:03:54 +0200 Subject: [PATCH 02/42] First draft of change log for v1.0.0rc1 --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bfbd8dd..ddbd4a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Change Log (Release Notes) All _notable_ changes to this project will be documented in this file (`CHANGELOG.md`). -This project adheres to [Semantic Versioning](http://semver.org/) (or at least we try). +This project adheres to [the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) (or at least we try). Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/). Note that each version (or "release") is the name of a [Git _tag_](https://git-scm.com/book/en/v2/Git-Basics-Tagging) of a particular commit, so the associated date and time are the date and time of that commit (as reported by GitHub), _not_ the "Uploaded on" date listed on PyPI (which may differ). For reference, the possible headings are: @@ -15,14 +15,71 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [1.0.0rc1] - 2017-06-?? +Tag name: v1.0.0rc1 + +### Added +* Support for SSL/TLS auth and communications between MongoDB and BigchainDB, MongoDB Backup Agent, MongoDB Monitoring Agent, and MongoDB Cloud Manager. Pull Requests +[#1456](https://github.com/bigchaindb/bigchaindb/pull/1456), +[#1497](https://github.com/bigchaindb/bigchaindb/pull/1497), +[#1510](https://github.com/bigchaindb/bigchaindb/pull/1510), +[#1536](https://github.com/bigchaindb/bigchaindb/pull/1536), and +[#1551](https://github.com/bigchaindb/bigchaindb/pull/1551). +* Text search support (only if using MongoDB). Pull Requests [#1469](https://github.com/bigchaindb/bigchaindb/pull/1469) and [#1471](https://github.com/bigchaindb/bigchaindb/pull/1471) +* The `database.connection_timeout` configuration setting now works with RethinkDB too. [#1512](https://github.com/bigchaindb/bigchaindb/pull/1512) +* New code and tools for benchmarking CREATE transactions. [Pull Request #1511](https://github.com/bigchaindb/bigchaindb/pull/1511) + + +### Changed +* There's an upgrade guide in `docs/upgrade-guides/v0.10-->v1.0.md`. It only covers changes to the transaction model and HTTP API. If that file hasn't been merged yet, see [Pull Request #1547](https://github.com/bigchaindb/bigchaindb/pull/1547) +* Cryptographic signatures now sign the whole (serialized) transaction body, including the transaction ID, but with all `"fulfillment"` values changed to `None`. [Pull Request #1225](https://github.com/bigchaindb/bigchaindb/pull/1225) +* In transactions, the value of `"amount"` must be a string. (Before, it was supposed to be a number.) [Pull Request #1286](https://github.com/bigchaindb/bigchaindb/pull/1286) +* In `setup.py`, the "Development Status" (as reported on PyPI) was changed from Alpha to Beta. [Pull Request #1437](https://github.com/bigchaindb/bigchaindb/pull/1437) +* If you explicitly specify a config file, e.g. `bigchaindb -c path/to/config start` and that file can't be found, then BigchainDB Server will fail with a helpful error message. [Pull Request #1486](https://github.com/bigchaindb/bigchaindb/pull/1486) +* Reduced the response time on the HTTP API endpoint to get all the unspent outputs associated with a given public key (a.k.a. "fast unspents"). [Pull Request #1411](https://github.com/bigchaindb/bigchaindb/pull/1411) +* Internally, the value of an asset's `"data"` is now stored in a separate assets table. This enabled the new text search. Each asset data is stored along with the associated CREATE transaction ID (asset ID). That data gets written when the containing block gets written to the bigchain table. [Pull Request #1460](https://github.com/bigchaindb/bigchaindb/pull/1460) +* Schema validation was sped up by switching to `rapidjson-schema`. [Pull Request #1494](https://github.com/bigchaindb/bigchaindb/pull/1494) +* If a node comes back from being down for a while, it will resume voting on blocks in the order determined by the MongoDB oplog, in the case of MongoDB. (In the case of RethinkDB, blocks missed in the changefeed will not be voted on.) [Pull Request #1389](https://github.com/bigchaindb/bigchaindb/pull/1389) +* Parallelized transaction schema validation in the vote pipeline. [Pull Request #1492](https://github.com/bigchaindb/bigchaindb/pull/1492) +* `asset.data` or `asset.id` are now *required* in a CREATE or TRANSFER transaction, respectively. [Pull Request #1518](https://github.com/bigchaindb/bigchaindb/pull/1518) +* The HTTP response body, in the response to the `GET /` and the `GET /api/v1` endpoints, was changed substantially. [Pull Request #1529](https://github.com/bigchaindb/bigchaindb/pull/1529) +* Changed the HTTP `GET /api/v1/transactions/{transaction_id}` endpoint. It now only returns the transaction if it's in a valid block. It also returns a new header with a relative link to a status monitor. [Pull Request #1543](https://github.com/bigchaindb/bigchaindb/pull/1543) +* All instances of `txid` and `tx_id` were replaced with `transaction_id`, in the transaction model and the HTTP API. [Pull Request #1532](https://github.com/bigchaindb/bigchaindb/pull/1532) +* The hostname and port were removed from all URLs in all HTTP API responses. [Pull Request #1538](https://github.com/bigchaindb/bigchaindb/pull/1538) +* Relative links were replaced with JSON objects in HTTP API responses. [Pull Request #1541](https://github.com/bigchaindb/bigchaindb/pull/1541) +* In the outputs endpoint of the HTTP API, the query parameter `unspent` was changed to `spent` (so no more double negatives). If that query parameter isn't included, then all outputs matching the specificed public key will be returned. If `spent=true`, then only the spent outputs will be returned. If `spent=false`, then only the unspent outputs will be returned. [Pull Request #1545](https://github.com/bigchaindb/bigchaindb/pull/1545) + + +### Removed +* The `server.threads` configuration setting (for the Gunicorn HTTP server) was removed from the default set of BigchainDB configuration settings. [Pull Request #1488](https://github.com/bigchaindb/bigchaindb/pull/1488) + + +### Fixed +* TODO: Pull Request #1450 fixed a bug but then it came back later? See [pull request #1470](https://github.com/bigchaindb/bigchaindb/pull/1470) ("#1450 unrevert") + + +### External Contributors +* @elopio - Pull Requests [#1415](https://github.com/bigchaindb/bigchaindb/pull/1415) and [#1491](https://github.com/bigchaindb/bigchaindb/pull/1491) +* @CsterKuroi - [Pull Request #1447](https://github.com/bigchaindb/bigchaindb/pull/1447) +* @tdsgit - [Pull Request #1512](https://github.com/bigchaindb/bigchaindb/pull/1512) +* @lavinasachdev3 - [Pull Request #1357](https://github.com/bigchaindb/bigchaindb/pull/1357) + + +### Notes +* There were many improvements to our Kubernetes-based production deployment template (and the associated documentaiton). +* There is now a [BigchainDB Ruby driver](https://github.com/LicenseRocks/bigchaindb_ruby), created by @addywaddy at [license.rocks](https://github.com/bigchaindb/bigchaindb/pull/1437). +* The [BigchainDB JavaScript driver](https://github.com/bigchaindb/js-bigchaindb-driver) was moved to a different GitHub repo and is now officially maintained by the BigchainDB team. +* We continue to recommend using MongoDB. + + ## [0.10.2] - 2017-05-16 Tag name: v0.10.2 -## Added +### Added * Add Cross Origin Resource Sharing (CORS) support for the HTTP API. [Commit 6cb7596](https://github.com/bigchaindb/bigchaindb/commit/6cb75960b05403c77bdae0fd327612482589efcb) -## Fixed +### Fixed * Fixed `streams_v1` API link in response to `GET /api/v1`. [Pull Request #1466](https://github.com/bigchaindb/bigchaindb/pull/1466) * Fixed mismatch between docs and implementation for `GET /blocks?status=` @@ -32,10 +89,10 @@ Tag name: v0.10.2 ## [0.10.1] - 2017-04-19 Tag name: v0.10.1 -## Added +### Added * Documentation for the BigchainDB settings `wsserver.host` and `wsserver.port`. [Pull Request #1408](https://github.com/bigchaindb/bigchaindb/pull/1408) -## Fixed +### Fixed * Fixed `Dockerfile`, which was failing to build. It now starts `FROM python:3.6` (instead of `FROM ubuntu:xenial`). [Pull Request #1410](https://github.com/bigchaindb/bigchaindb/pull/1410) * Fixed the `Makefile` so that `release` depends on `dist`. [Pull Request #1405](https://github.com/bigchaindb/bigchaindb/pull/1405) From 8f46f51e977f1961bac75dc0b35276b0f51ebb38 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 16:14:51 +0200 Subject: [PATCH 03/42] Made CHANGELOG edit suggested by @krish7919 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddbd4a4e..ebb0ba2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ For reference, the possible headings are: Tag name: v1.0.0rc1 ### Added -* Support for SSL/TLS auth and communications between MongoDB and BigchainDB, MongoDB Backup Agent, MongoDB Monitoring Agent, and MongoDB Cloud Manager. Pull Requests +* Support for secure TLS/SSL communication between MongoDB and {BigchainDB, MongoDB Backup Agent, MongoDB Monitoring Agent}. Pull Requests [#1456](https://github.com/bigchaindb/bigchaindb/pull/1456), [#1497](https://github.com/bigchaindb/bigchaindb/pull/1497), [#1510](https://github.com/bigchaindb/bigchaindb/pull/1510), From 2c53dfcd57e7b5fd715f2f8f464680b9da105ec1 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 16:20:44 +0200 Subject: [PATCH 04/42] Added note re dropping support for Python 3.4 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebb0ba2b..5d0f64d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ Tag name: v1.0.0rc1 ### Notes +* We dropped support for Python 3.4. [Pull Request #1564](https://github.com/bigchaindb/bigchaindb/pull/1564) * There were many improvements to our Kubernetes-based production deployment template (and the associated documentaiton). * There is now a [BigchainDB Ruby driver](https://github.com/LicenseRocks/bigchaindb_ruby), created by @addywaddy at [license.rocks](https://github.com/bigchaindb/bigchaindb/pull/1437). * The [BigchainDB JavaScript driver](https://github.com/bigchaindb/js-bigchaindb-driver) was moved to a different GitHub repo and is now officially maintained by the BigchainDB team. From 9e247f9318fc1b3a4531ff7ec66c2520555d9450 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 17:22:40 +0200 Subject: [PATCH 05/42] Removed all mentions of BIGCHAINDB_SERVER_THREADS --- docs/server/source/server-reference/configuration.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 9766688e..1674f6e5 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -21,7 +21,6 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_SERVER_BIND`
`BIGCHAINDB_SERVER_LOGLEVEL`
`BIGCHAINDB_SERVER_WORKERS`
-`BIGCHAINDB_SERVER_THREADS`
`BIGCHAINDB_WSSERVER_SCHEME`
`BIGCHAINDB_WSSERVER_HOST`
`BIGCHAINDB_WSSERVER_PORT`
@@ -189,7 +188,6 @@ for more information. export BIGCHAINDB_SERVER_BIND=0.0.0.0:9984 export BIGCHAINDB_SERVER_LOGLEVEL=debug export BIGCHAINDB_SERVER_WORKERS=5 -export BIGCHAINDB_SERVER_THREADS=5 ``` **Example config file snippet** From 3aeb20afd3741f562f3a4f7d1efe17bb2aa98f5e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 17:27:51 +0200 Subject: [PATCH 06/42] Changed some ticks to backticks in configuration.md --- docs/server/source/server-reference/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 1674f6e5..f0ce46dc 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -175,7 +175,7 @@ If you used `bigchaindb -y configure mongodb` to create a default local config f These settings are for the [Gunicorn HTTP server](http://gunicorn.org/), which is used to serve the [HTTP client-server API](../http-client-server-api.html). -`server.bind` is where to bind the Gunicorn HTTP server socket. It's a string. It can be any valid value for [Gunicorn's bind setting](http://docs.gunicorn.org/en/stable/settings.html#bind). If you want to allow IPv4 connections from anyone, on port 9984, use '0.0.0.0:9984'. In a production setting, we recommend you use Gunicorn behind a reverse proxy server. If Gunicorn and the reverse proxy are running on the same machine, then use 'localhost:PORT' where PORT is _not_ 9984 (because the reverse proxy needs to listen on port 9984). Maybe use PORT=9983 in that case because we know 9983 isn't used. If Gunicorn and the reverse proxy are running on different machines, then use 'A.B.C.D:9984' where A.B.C.D is the IP address of the reverse proxy. There's [more information about deploying behind a reverse proxy in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) +`server.bind` is where to bind the Gunicorn HTTP server socket. It's a string. It can be any valid value for [Gunicorn's bind setting](http://docs.gunicorn.org/en/stable/settings.html#bind). If you want to allow IPv4 connections from anyone, on port 9984, use `0.0.0.0:9984`. In a production setting, we recommend you use Gunicorn behind a reverse proxy server. If Gunicorn and the reverse proxy are running on the same machine, then use `localhost:PORT` where PORT is _not_ 9984 (because the reverse proxy needs to listen on port 9984). Maybe use PORT=9983 in that case because we know 9983 isn't used. If Gunicorn and the reverse proxy are running on different machines, then use `A.B.C.D:9984` where A.B.C.D is the IP address of the reverse proxy. There's [more information about deploying behind a reverse proxy in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) `server.loglevel` sets the log level of Gunicorn's Error log outputs. See [Gunicorn's documentation](http://docs.gunicorn.org/en/latest/settings.html#loglevel) From 3229e2e26244b6c37d69b4533f6b6d575c9e75c2 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 17:32:22 +0200 Subject: [PATCH 07/42] =?UTF-8?q?docs:=20changed=20the=20*=20symbol=20to?= =?UTF-8?q?=20=C3=97=20for=20multiplication.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/server/source/server-reference/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index f0ce46dc..2ec4bf04 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -181,7 +181,7 @@ These settings are for the [Gunicorn HTTP server](http://gunicorn.org/), which i [Gunicorn's documentation](http://docs.gunicorn.org/en/latest/settings.html#loglevel) for more information. -`server.workers` is [the number of worker processes](http://docs.gunicorn.org/en/stable/settings.html#workers) for handling requests. If `None` (the default), the value will be (cpu_count * 2 + 1). Each worker process has a single thread. The HTTP server will be able to handle `server.workers` requests simultaneously. +`server.workers` is [the number of worker processes](http://docs.gunicorn.org/en/stable/settings.html#workers) for handling requests. If `None` (the default), the value will be (2 × cpu_count + 1). Each worker process has a single thread. The HTTP server will be able to handle `server.workers` requests simultaneously. **Example using environment variables** ```text From 3ca11c1f6936a8e3bd19bb473cd32b6bbbd24f3c Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 20 Jun 2017 17:40:08 +0200 Subject: [PATCH 08/42] Emphasize MongoDB over RethinkDB in configuration.md --- docs/server/source/server-reference/configuration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 2ec4bf04..9a86f3e4 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -51,7 +51,7 @@ Note that the `-c` command line option will always take precedence if both the ` You can read the current default values in the file [bigchaindb/\_\_init\_\_.py](https://github.com/bigchaindb/bigchaindb/blob/master/bigchaindb/__init__.py). (The link is to the latest version.) -Running `bigchaindb -y configure rethinkdb` will generate a local config file in `$HOME/.bigchaindb` with all the default values, with two exceptions: It will generate a valid private/public keypair, rather than using the default keypair (`None` and `None`). +Running `bigchaindb -y configure mongodb` will generate a local config file in `$HOME/.bigchaindb` with all the default values (for using MongoDB as the database backend), with two exceptions: it will generate a valid private/public keypair, rather than using the default keypair (`None` and `None`). ## keypair.public & keypair.private @@ -72,7 +72,7 @@ export BIGCHAINDB_KEYPAIR_PRIVATE=5C5Cknco7YxBRP9AgB1cbUVTL4FAcooxErLygw1DeG2D } ``` -Internally (i.e. in the Python code), both keys have a default value of `None`, but that's not a valid key. Therefore you can't rely on the defaults for the keypair. If you want to run BigchainDB, you must provide a valid keypair, either in the environment variables or in the local config file. You can generate a local config file with a valid keypair (and default everything else) using `bigchaindb -y configure rethinkdb`. +Internally (i.e. in the Python code), both keys have a default value of `None`, but that's not a valid key. Therefore you can't rely on the defaults for the keypair. If you want to run BigchainDB, you must provide a valid keypair, either in the environment variables or in the local config file. You can generate a local config file with a valid keypair (and default everything else) using `bigchaindb -y configure mongodb`. ## keyring @@ -101,12 +101,12 @@ Note how the keys in the list are separated by colons. ## database.* The settings with names of the form `database.*` are for the database backend -(currently either RethinkDB or MongoDB). They are: +(currently either MongoDB or RethinkDB). They are: -* `database.backend` is either `rethinkdb` or `mongodb`. +* `database.backend` is either `mongodb` or `rethinkdb`. * `database.host` is the hostname (FQDN) of the backend database. * `database.port` is self-explanatory. -* `database.name` is a user-chosen name for the database inside RethinkDB or MongoDB, e.g. `bigchain`. +* `database.name` is a user-chosen name for the database inside MongoDB or RethinkDB, e.g. `bigchain`. * `database.replicaset` is only relevant if using MongoDB; it's the name of the MongoDB replica set, e.g. `bigchain-rs`. * `database.connection_timeout` is the maximum number of milliseconds that BigchainDB will wait before giving up on one attempt to connect to the database backend. * `database.max_tries` is the maximum number of times that BigchainDB will try to establish a connection with the database backend. If 0, then it will try forever. From 31e3dd224c7ee2981c8e1ce77fd3eeaaff9dd07a Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 22 Jun 2017 15:10:10 +0200 Subject: [PATCH 09/42] Note re PR #1450 in CHANGELOG --- CHANGELOG.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d0f64d5..a2300165 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,6 @@ Tag name: v1.0.0rc1 * The `database.connection_timeout` configuration setting now works with RethinkDB too. [#1512](https://github.com/bigchaindb/bigchaindb/pull/1512) * New code and tools for benchmarking CREATE transactions. [Pull Request #1511](https://github.com/bigchaindb/bigchaindb/pull/1511) - ### Changed * There's an upgrade guide in `docs/upgrade-guides/v0.10-->v1.0.md`. It only covers changes to the transaction model and HTTP API. If that file hasn't been merged yet, see [Pull Request #1547](https://github.com/bigchaindb/bigchaindb/pull/1547) * Cryptographic signatures now sign the whole (serialized) transaction body, including the transaction ID, but with all `"fulfillment"` values changed to `None`. [Pull Request #1225](https://github.com/bigchaindb/bigchaindb/pull/1225) @@ -49,14 +48,11 @@ Tag name: v1.0.0rc1 * Relative links were replaced with JSON objects in HTTP API responses. [Pull Request #1541](https://github.com/bigchaindb/bigchaindb/pull/1541) * In the outputs endpoint of the HTTP API, the query parameter `unspent` was changed to `spent` (so no more double negatives). If that query parameter isn't included, then all outputs matching the specificed public key will be returned. If `spent=true`, then only the spent outputs will be returned. If `spent=false`, then only the unspent outputs will be returned. [Pull Request #1545](https://github.com/bigchaindb/bigchaindb/pull/1545) - ### Removed * The `server.threads` configuration setting (for the Gunicorn HTTP server) was removed from the default set of BigchainDB configuration settings. [Pull Request #1488](https://github.com/bigchaindb/bigchaindb/pull/1488) - ### Fixed -* TODO: Pull Request #1450 fixed a bug but then it came back later? See [pull request #1470](https://github.com/bigchaindb/bigchaindb/pull/1470) ("#1450 unrevert") - +* The `GET /api/v1/outputs` endpoint was failing for some transactions with threshold conditions. Fixed in [Pull Request #1450](https://github.com/bigchaindb/bigchaindb/pull/1450) ### External Contributors * @elopio - Pull Requests [#1415](https://github.com/bigchaindb/bigchaindb/pull/1415) and [#1491](https://github.com/bigchaindb/bigchaindb/pull/1491) @@ -64,7 +60,6 @@ Tag name: v1.0.0rc1 * @tdsgit - [Pull Request #1512](https://github.com/bigchaindb/bigchaindb/pull/1512) * @lavinasachdev3 - [Pull Request #1357](https://github.com/bigchaindb/bigchaindb/pull/1357) - ### Notes * We dropped support for Python 3.4. [Pull Request #1564](https://github.com/bigchaindb/bigchaindb/pull/1564) * There were many improvements to our Kubernetes-based production deployment template (and the associated documentaiton). From 4277c6b8573c5bdc416fb3ed3b84b23d0b5631a3 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 22 Jun 2017 15:14:06 +0200 Subject: [PATCH 10/42] Added note re switching to crypto-conditions version 02 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2300165..4e4dc6a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ Tag name: v1.0.0rc1 * The hostname and port were removed from all URLs in all HTTP API responses. [Pull Request #1538](https://github.com/bigchaindb/bigchaindb/pull/1538) * Relative links were replaced with JSON objects in HTTP API responses. [Pull Request #1541](https://github.com/bigchaindb/bigchaindb/pull/1541) * In the outputs endpoint of the HTTP API, the query parameter `unspent` was changed to `spent` (so no more double negatives). If that query parameter isn't included, then all outputs matching the specificed public key will be returned. If `spent=true`, then only the spent outputs will be returned. If `spent=false`, then only the unspent outputs will be returned. [Pull Request #1545](https://github.com/bigchaindb/bigchaindb/pull/1545) +* The supported crypto-conditions changed from version 01 of the crypto-conditions spec to version 02. [Pull Request #1562](https://github.com/bigchaindb/bigchaindb/pull/1562) ### Removed * The `server.threads` configuration setting (for the Gunicorn HTTP server) was removed from the default set of BigchainDB configuration settings. [Pull Request #1488](https://github.com/bigchaindb/bigchaindb/pull/1488) From 35c3909c254ee46b752ce0e32d1d002e559888c5 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 22 Jun 2017 16:32:57 +0200 Subject: [PATCH 11/42] Noted PR #1552 in the CHANGELOG --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e4dc6a7..71f86a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,9 @@ Tag name: v1.0.0rc1 [#1456](https://github.com/bigchaindb/bigchaindb/pull/1456), [#1497](https://github.com/bigchaindb/bigchaindb/pull/1497), [#1510](https://github.com/bigchaindb/bigchaindb/pull/1510), -[#1536](https://github.com/bigchaindb/bigchaindb/pull/1536), and -[#1551](https://github.com/bigchaindb/bigchaindb/pull/1551). +[#1536](https://github.com/bigchaindb/bigchaindb/pull/1536), +[#1551](https://github.com/bigchaindb/bigchaindb/pull/1551) and +[#1552](https://github.com/bigchaindb/bigchaindb/pull/1552). * Text search support (only if using MongoDB). Pull Requests [#1469](https://github.com/bigchaindb/bigchaindb/pull/1469) and [#1471](https://github.com/bigchaindb/bigchaindb/pull/1471) * The `database.connection_timeout` configuration setting now works with RethinkDB too. [#1512](https://github.com/bigchaindb/bigchaindb/pull/1512) * New code and tools for benchmarking CREATE transactions. [Pull Request #1511](https://github.com/bigchaindb/bigchaindb/pull/1511) From 3efac22b3c6202540557b5af17af6738a1903fb3 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 22 Jun 2017 16:52:22 +0200 Subject: [PATCH 12/42] CHANGELOG: Added note how 'version' must be '1.0' in transactions --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f86a18..ae0590ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Tag name: v1.0.0rc1 * Relative links were replaced with JSON objects in HTTP API responses. [Pull Request #1541](https://github.com/bigchaindb/bigchaindb/pull/1541) * In the outputs endpoint of the HTTP API, the query parameter `unspent` was changed to `spent` (so no more double negatives). If that query parameter isn't included, then all outputs matching the specificed public key will be returned. If `spent=true`, then only the spent outputs will be returned. If `spent=false`, then only the unspent outputs will be returned. [Pull Request #1545](https://github.com/bigchaindb/bigchaindb/pull/1545) * The supported crypto-conditions changed from version 01 of the crypto-conditions spec to version 02. [Pull Request #1562](https://github.com/bigchaindb/bigchaindb/pull/1562) +* The value of "version" inside a transaction must now be "1.0". (Before, it could be "0.anything".) [Pull Request #1574](https://github.com/bigchaindb/bigchaindb/pull/1574) ### Removed * The `server.threads` configuration setting (for the Gunicorn HTTP server) was removed from the default set of BigchainDB configuration settings. [Pull Request #1488](https://github.com/bigchaindb/bigchaindb/pull/1488) From 47252eabc0de6512a1c9cf48b8d19dcf0a456074 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 23 Jun 2017 09:28:05 +0200 Subject: [PATCH 13/42] Set the release date of v1.0.0rc1 to 2017-06-23 in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae0590ab..14ca4ff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** -## [1.0.0rc1] - 2017-06-?? +## [1.0.0rc1] - 2017-06-23 Tag name: v1.0.0rc1 ### Added From e74e32d6eab2ee010eeb5642dd193ddd79834505 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 23 Jun 2017 09:50:27 +0200 Subject: [PATCH 14/42] Bump version to 1.0.0rc1 --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 6bf027a0..d7f2fa31 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.11.0.dev' -__short_version__ = '0.11.dev' +__version__ = '1.0.0rc1' +__short_version__ = '1.0rc1' From 190a62322636ad1194e19c0dcf37d87d02e1b49d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 23 Jun 2017 09:57:42 +0200 Subject: [PATCH 15/42] Update kubernetes docker image version --- k8s/bigchaindb/bigchaindb-dep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 80fcf0a1..376c5c86 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:0.10.2 + image: bigchaindb/bigchaindb:1.0.0rc1 imagePullPolicy: IfNotPresent args: - start From 3aab15d398a43ed28de75c89b59e2abdfb30cf92 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 23 Jun 2017 10:59:14 +0200 Subject: [PATCH 16/42] Switch version to development mode --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index d7f2fa31..ee90f2c4 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '1.0.0rc1' -__short_version__ = '1.0rc1' +__version__ = '1.0.0.dev' +__short_version__ = '1.0.dev' From 164cd47225dc58f35d1511a4e53644d14b1fa01e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 23 Jun 2017 11:47:22 +0200 Subject: [PATCH 17/42] Note re Python form of semantic versioning in Release Process doc --- Release_Process.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index be4c448a..cf8036dd 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -2,8 +2,14 @@ The release process for BigchainDB server differs slightly depending on whether it's a minor or a patch release. -BigchainDB follows [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH), taking into account -that [major version 0.x does not export a stable API](http://semver.org/#spec-item-4). +BigchainDB follows +[the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) +(i.e. MAJOR.MINOR.PATCH), +which is almost identical +to [regular semantic versioning](http://semver.org/) +except release candidates are labelled like +`3.4.5rc2` not `3.4.5-rc2` (with no hyphen). + ## Minor release From 39a200e9ddf3ada6223e70e188624f17c4b3bd38 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 23 Jun 2017 11:49:18 +0200 Subject: [PATCH 18/42] Note re creating PR on GitHub & waiting for tests to pass --- Release_Process.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Release_Process.md b/Release_Process.md index cf8036dd..a12756a0 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -20,6 +20,7 @@ A minor release is preceeded by a feature freeze and created from the 'master' b 1. Create and checkout a new branch for the minor release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` (*not* 0.9.0, this new branch will be for e.g. 0.9.0, 0.9.1, 0.9.2, etc. each of which will be identified by a tagged commit) 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) 1. Commit that change, and push the new branch to GitHub +1. On GitHub, use the new branch to create a new pull request and wait for all the tests to pass 1. Follow steps outlined in [Common Steps](#common-steps) 1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev`. This is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available). 1. Go to [Docker Hub](https://hub.docker.com/), sign in, go to Settings - Build Settings, and under the build with Docker Tag Name equal to `latest`, change the Name to the number of the new release, e.g. `0.9` From 998ff34c4b144fd8516e0db56112f6e04426fe2e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 23 Jun 2017 11:51:00 +0200 Subject: [PATCH 19/42] Renamed Release_Process.md to RELEASE_PROCESS.md --- Release_Process.md => RELEASE_PROCESS.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Release_Process.md => RELEASE_PROCESS.md (100%) diff --git a/Release_Process.md b/RELEASE_PROCESS.md similarity index 100% rename from Release_Process.md rename to RELEASE_PROCESS.md From e91e3a79c4c7fc27a51a29a4047882fdbe68bf28 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 26 Jun 2017 15:17:56 +0200 Subject: [PATCH 20/42] Made Easy-RSA config steps clearer. --- .../easy-rsa.rst | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/server/source/production-deployment-template/easy-rsa.rst b/docs/server/source/production-deployment-template/easy-rsa.rst index 3c87fbbc..3964d362 100644 --- a/docs/server/source/production-deployment-template/easy-rsa.rst +++ b/docs/server/source/production-deployment-template/easy-rsa.rst @@ -66,12 +66,17 @@ The comments in the file explain what each of the variables mean. echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> vars echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> vars echo 'set_var EASYRSA_REQ_ORG "BigchainDB GmbH"' >> vars - echo 'set_var EASYRSA_REQ_OU "IT"' >> vars echo 'set_var EASYRSA_REQ_EMAIL "dev@bigchaindb.com"' >> vars -We follow the convention of modifying the OU to ``ROOT-CA``, -``MongoDB-Instance``, ``BigchainDB-Instance``, ``MongoDB-Mon-Instance`` and -``MongoDB-Backup-Instance`` while issuing certificates. +We follow the convention of setting the OU to ``ROOT-CA``, +``MongoDB-Instance``, ``BigchainDB-Instance``, ``MongoDB-Mon-Instance`` or +``MongoDB-Backup-Instance`` as appropriate. +Replace ``insert-name-here`` with the appropriate name +(e.g. ``ROOT-CA``) in: + +.. code:: bash + + echo 'set_var EASYRSA_REQ_OU "insert-name-here"' >> vars Step 4: Maybe Edit x509-types/server @@ -81,7 +86,7 @@ Step 4: Maybe Edit x509-types/server Only do this step if you are setting up a self-signed CA. -Edit the file ``x509-types/server`` and change -``extendedKeyUsage = serverAuth`` to -``extendedKeyUsage = serverAuth,clientAuth``. -See `the MongoDB documentation about x.509 authentication `_ to understand why. + Edit the file ``x509-types/server`` and change + ``extendedKeyUsage = serverAuth`` to + ``extendedKeyUsage = serverAuth,clientAuth``. + See `the MongoDB documentation about x.509 authentication `_ to understand why. From 6fd5f216387ec222fa421ef6738701865282b444 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 26 Jun 2017 15:34:03 +0200 Subject: [PATCH 21/42] Improved instructions for Self-Signed CA setup --- .../production-deployment-template/ca-installation.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/server/source/production-deployment-template/ca-installation.rst b/docs/server/source/production-deployment-template/ca-installation.rst index 4c6af317..aad945d3 100644 --- a/docs/server/source/production-deployment-template/ca-installation.rst +++ b/docs/server/source/production-deployment-template/ca-installation.rst @@ -33,15 +33,12 @@ by going to the ``bdb-cluster-ca/easy-rsa-3.0.1/easyrsa3`` directory and using: ./easyrsa build-ca -You will be prompted to enter the Distinguished Name for this CA. You can hit -enter to accept the default values or change it at each prompt. - -You will also be asked to enter a PEM pass phrase for encrypting the ``ca.key`` file. +You will also be asked to enter a PEM pass phrase (for encrypting the ``ca.key`` file). Make sure to securely store that PEM pass phrase. If you lose it, you won't be able to add or remove entities from your PKI infrastructure in the future. -It will ask several other questions. -You can accept all the defaults [in brackets] by pressing Enter. +You will be prompted to enter the Distinguished Name (DN) information for this CA. +For each field, you can accept the default value [in brackets] by pressing Enter. While ``Easy-RSA CA`` *is* a valid and acceptable Common Name, you should probably enter a name based on the name of the managing organization, e.g. ``Omega Ledger CA``. From 3a24e7d9b40b6ddc4ceadbefe39d5b40e9049d44 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 26 Jun 2017 15:54:09 +0200 Subject: [PATCH 22/42] List what the self-signed CA should return to the node operator --- .../source/production-deployment-template/workflow.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/server/source/production-deployment-template/workflow.rst b/docs/server/source/production-deployment-template/workflow.rst index db3ff0b3..fb80c5fe 100644 --- a/docs/server/source/production-deployment-template/workflow.rst +++ b/docs/server/source/production-deployment-template/workflow.rst @@ -119,9 +119,13 @@ allow easier periodic rotation of the ``Agent API Key`` with a constant #. Client Certificate for BigchainDB Server to identify itself to MongoDB #. Client Certificate for MongoDB Monitoring Agent to identify itself to MongoDB #. Client Certificate for MongoDB Backup Agent to identify itself to MongoDB -#. CRL for the infrastructure to not accept revoked certificates. -Ask the managing organization to use its self-signed CA to sign those certificates. +Ask the managing organization to use its self-signed CA to sign those four certificates. +They should send you: + +* Signed versions of your four certificates. +* One ``ca.crt`` file: their CA certificate. +* One ``crl.pem`` file: a certificate revocation list. For help, see the pages: From 1a1984f5452308617f4d5a8140509dc3af5c6e30 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 27 Jun 2017 11:15:40 +0200 Subject: [PATCH 23/42] More clarifications to the MongoDB SSL cert instructions --- .../ca-installation.rst | 5 ++ .../client-tls-certificate.rst | 28 +++++--- .../easy-rsa.rst | 14 ++-- .../revoke-tls-certificate.rst | 2 + .../server-tls-certificate.rst | 25 ++++--- .../workflow.rst | 71 +++++++------------ 6 files changed, 72 insertions(+), 73 deletions(-) diff --git a/docs/server/source/production-deployment-template/ca-installation.rst b/docs/server/source/production-deployment-template/ca-installation.rst index aad945d3..146bd461 100644 --- a/docs/server/source/production-deployment-template/ca-installation.rst +++ b/docs/server/source/production-deployment-template/ca-installation.rst @@ -39,6 +39,11 @@ If you lose it, you won't be able to add or remove entities from your PKI infras You will be prompted to enter the Distinguished Name (DN) information for this CA. For each field, you can accept the default value [in brackets] by pressing Enter. + +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value ``ROOT-CA``. + While ``Easy-RSA CA`` *is* a valid and acceptable Common Name, you should probably enter a name based on the name of the managing organization, e.g. ``Omega Ledger CA``. diff --git a/docs/server/source/production-deployment-template/client-tls-certificate.rst b/docs/server/source/production-deployment-template/client-tls-certificate.rst index 9fd748bd..5a729836 100644 --- a/docs/server/source/production-deployment-template/client-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/client-tls-certificate.rst @@ -24,7 +24,7 @@ Step 2: Create the Client Private Key and CSR --------------------------------------------- You can create the client private key and certificate signing request (CSR) -by going into the directory ``client-cert/easy-rsa-3.0.1/easyrsa`` +by going into the directory ``client-cert/easy-rsa-3.0.1/easyrsa3`` and using: .. code:: bash @@ -33,25 +33,37 @@ and using: ./easyrsa gen-req bdb-instance-0 nopass -You should change ``bdb-instance-0`` to a value that reflects what the -client certificate is being used for. +You should change the Common Name (e.g. ``bdb-instance-0``) +to a value that reflects what the +client certificate is being used for, e.g. ``mdb-mon-instance-3`` or ``mdb-bak-instance-4``. (The final integer is specific to your BigchainDB node in the BigchainDB cluster.) -Tip: You can get help with the ``easyrsa`` command (and its subcommands) -by using the subcommand ``./easyrsa help`` +You will be prompted to enter the Distinguished Name (DN) information for this certificate. For each field, you can accept the default value [in brackets] by pressing Enter. + +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value + ``BigchainDB-Instance``, ``MongoDB-Mon-Instance`` or ``MongoDB-Backup-Instance`` + as appropriate. + +Aside: The ``nopass`` option means "do not encrypt the private key (default is encrypted)". You can get help with the ``easyrsa`` command (and its subcommands) +by using the subcommand ``./easyrsa help``. Step 3: Get the Client Certificate Signed ----------------------------------------- -The CSR file (created in the previous step) -should be located in ``pki/reqs/bdb-instance-0.req``. +The CSR file created in the previous step +should be located in ``pki/reqs/bdb-instance-0.req`` +(or whatever Common Name you used in the ``gen-req`` command above). You need to send it to the organization managing the cluster so that they can use their CA to sign the request. (The managing organization should already have a self-signed CA.) If you are the admin of the managing organization's self-signed CA, -then you can import the CSR and use Easy-RSA to sign it. For example: +then you can import the CSR and use Easy-RSA to sign it. +Go to your ``bdb-cluster-ca/easy-rsa-3.0.1/easyrsa3/`` +directory and do something like: .. code:: bash diff --git a/docs/server/source/production-deployment-template/easy-rsa.rst b/docs/server/source/production-deployment-template/easy-rsa.rst index 3964d362..ff268bf2 100644 --- a/docs/server/source/production-deployment-template/easy-rsa.rst +++ b/docs/server/source/production-deployment-template/easy-rsa.rst @@ -66,17 +66,15 @@ The comments in the file explain what each of the variables mean. echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> vars echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> vars echo 'set_var EASYRSA_REQ_ORG "BigchainDB GmbH"' >> vars + echo 'set_var EASYRSA_REQ_OU "IT"' >> vars echo 'set_var EASYRSA_REQ_EMAIL "dev@bigchaindb.com"' >> vars -We follow the convention of setting the OU to ``ROOT-CA``, +Note: Later, when building a CA or generating a certificate signing request, you will be prompted to enter a value for the OU (or to accept the default). You should change the default OU from ``IT`` to one of the following, as appropriate: +``ROOT-CA``, ``MongoDB-Instance``, ``BigchainDB-Instance``, ``MongoDB-Mon-Instance`` or -``MongoDB-Backup-Instance`` as appropriate. -Replace ``insert-name-here`` with the appropriate name -(e.g. ``ROOT-CA``) in: - -.. code:: bash - - echo 'set_var EASYRSA_REQ_OU "insert-name-here"' >> vars +``MongoDB-Backup-Instance``. +To understand why, see `the MongoDB Manual `_. +There are reminders to do this in the relevant docs. Step 4: Maybe Edit x509-types/server diff --git a/docs/server/source/production-deployment-template/revoke-tls-certificate.rst b/docs/server/source/production-deployment-template/revoke-tls-certificate.rst index a0e06373..7584ceb5 100644 --- a/docs/server/source/production-deployment-template/revoke-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/revoke-tls-certificate.rst @@ -39,3 +39,5 @@ Generate a new CRL for your infrastructure using: The generated ``crl.pem`` file needs to be uploaded to your infrastructure to prevent the revoked certificate from being used again. + +In particlar, the generated ``crl.pem`` file should be sent to all BigchainDB node operators in your BigchainDB cluster, so that they can update it in their MongoDB instance and their BigchainDB Server instance. diff --git a/docs/server/source/production-deployment-template/server-tls-certificate.rst b/docs/server/source/production-deployment-template/server-tls-certificate.rst index 844f670a..eb9dd204 100644 --- a/docs/server/source/production-deployment-template/server-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/server-tls-certificate.rst @@ -26,7 +26,7 @@ Step 2: Create the Server Private Key and CSR --------------------------------------------- You can create the server private key and certificate signing request (CSR) -by going into the directory ``member-cert/easy-rsa-3.0.1/easyrsa`` +by going into the directory ``member-cert/easy-rsa-3.0.1/easyrsa3`` and using something like: .. code:: bash @@ -35,15 +35,17 @@ and using something like: ./easyrsa --req-cn=mdb-instance-0 --subject-alt-name=DNS:localhost,DNS:mdb-instance-0 gen-req mdb-instance-0 nopass -You will be prompted to enter the Distinguished Name for this certificate. You -can hit enter to accept the default values or change them at each prompt. +You should replace the Common Name (``mdb-instance-0`` above) with the correct name for *your* MongoDB instance in the cluster, e.g. ``mdb-instance-5`` or ``mdb-instance-12``. (This name is decided by the organization managing the cluster.) -You can replace the common name (``mdb-instance-0`` above) with any other name -so long as the instance can verify that it is the hostname. +You will be prompted to enter the Distinguished Name (DN) information for this certificate. +For each field, you can accept the default value [in brackets] by pressing Enter. -You need to provide the ``DNS:localhost`` SAN during certificate generation +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value ``MongoDB-Instance``. + +Aside: You need to provide the ``DNS:localhost`` SAN during certificate generation for using the ``localhost exception`` in the MongoDB instance. - All certificates can have this attribute without compromising security as the ``localhost exception`` works only the first time. @@ -51,15 +53,18 @@ All certificates can have this attribute without compromising security as the Step 3: Get the Server Certificate Signed ----------------------------------------- -The CSR file (created in the last step) -should be located in ``pki/reqs/mdb-instance-0.req``. +The CSR file created in the last step +should be located in ``pki/reqs/mdb-instance-0.req`` +(where the integer ``0`` may be different for you). You need to send it to the organization managing the cluster so that they can use their CA to sign the request. (The managing organization should already have a self-signed CA.) If you are the admin of the managing organization's self-signed CA, -then you can import the CSR and use Easy-RSA to sign it. For example: +then you can import the CSR and use Easy-RSA to sign it. +Go to your ``bdb-cluster-ca/easy-rsa-3.0.1/easyrsa3/`` +directory and do something like: .. code:: bash diff --git a/docs/server/source/production-deployment-template/workflow.rst b/docs/server/source/production-deployment-template/workflow.rst index fb80c5fe..c511b8f9 100644 --- a/docs/server/source/production-deployment-template/workflow.rst +++ b/docs/server/source/production-deployment-template/workflow.rst @@ -53,6 +53,26 @@ Similarly, other instances must also have unique names in the cluster. #. Name of the MongoDB backup agent instance (``mdb-bak-instance-*``) +☐ Generate four keys and corresponding certificate signing requests (CSRs): + +#. Server Certificate (a.k.a. Member Certificate) for the MongoDB instance +#. Client Certificate for BigchainDB Server to identify itself to MongoDB +#. Client Certificate for MongoDB Monitoring Agent to identify itself to MongoDB +#. Client Certificate for MongoDB Backup Agent to identify itself to MongoDB + +Ask the managing organization to use its self-signed CA to sign those four CSRs. +They should send you: + +* Four certificates (one for each CSR you sent them). +* One ``ca.crt`` file: their CA certificate. +* One ``crl.pem`` file: a certificate revocation list. + +For help, see the pages: + +* :ref:`How to Generate a Server Certificate for MongoDB` +* :ref:`How to Generate a Client Certificate for MongoDB` + + ☐ Every node in a BigchainDB cluster needs its own BigchainDB keypair (i.e. a public key and corresponding private key). You can generate a BigchainDB keypair for your node, for example, @@ -73,28 +93,15 @@ Don't share your private key. That list of public keys is known as the BigchainDB "keyring." -☐ Ask the managing organization -for the FQDN used to serve the BigchainDB APIs -(e.g. ``api.orgname.net`` or ``bdb.clustername.com``). - - ☐ Make up an FQDN for your BigchainDB node (e.g. ``mynode.mycorp.com``). Make sure you've registered the associated domain name (e.g. ``mycorp.com``), and have an SSL certificate for the FQDN. -(You can get an SSL certificate from any SSL certificate provider). - - -☐ Share your BigchaindB *public* key with all the other nodes -in the BigchainDB cluster. -Don't share your private key. - - -☐ Get the BigchainDB public keys of all the other nodes in the cluster. -That list of public keys is known as the BigchainDB "keyring." +(You can get an SSL certificate from any SSL certificate provider.) ☐ Ask the managing organization for the FQDN used to serve the BigchainDB APIs +(e.g. ``api.orgname.net`` or ``bdb.clustername.com``) and for a copy of the associated SSL/TLS certificate. Also, ask for the user name to use for authenticating to MongoDB. @@ -113,41 +120,11 @@ allow easier periodic rotation of the ``Agent API Key`` with a constant ``Group ID``) -☐ Generate four keys and corresponding certificate signing requests (CSRs): - -#. Server Certificate (a.k.a. Member Certificate) for the MongoDB instance -#. Client Certificate for BigchainDB Server to identify itself to MongoDB -#. Client Certificate for MongoDB Monitoring Agent to identify itself to MongoDB -#. Client Certificate for MongoDB Backup Agent to identify itself to MongoDB - -Ask the managing organization to use its self-signed CA to sign those four certificates. -They should send you: - -* Signed versions of your four certificates. -* One ``ca.crt`` file: their CA certificate. -* One ``crl.pem`` file: a certificate revocation list. - -For help, see the pages: - -* :ref:`How to Generate a Server Certificate for MongoDB` -* :ref:`How to Generate a Client Certificate for MongoDB` - - ☐ :doc:`Deploy a Kubernetes cluster on Azure `. -☐ Create the Kubernetes Configuration for this node. -We will use Kubernetes ConfigMaps and Secrets to hold all the information -gathered above. - - -☐ Deploy your BigchainDB node on your Kubernetes cluster. - -Next Steps To Set Up a Node ---------------------------- - -You can now proceed to set up your BigchainDB node based on whether it is the -:ref:`first node in you cluster +☐ You can now proceed to set up your BigchainDB node based on whether it is the +:ref:`first node in a new cluster ` or a :ref:`node that will be added to an existing cluster `. From 69a07b282db17b681e5056b8424c1243a9200cbd Mon Sep 17 00:00:00 2001 From: libscott Date: Tue, 27 Jun 2017 15:27:49 -0700 Subject: [PATCH 24/42] remove federation test that is failing sometimes in the case of RethinkDB (#1587) --- tests/integration/test_federation.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/integration/test_federation.py b/tests/integration/test_federation.py index 00c59685..598412ff 100644 --- a/tests/integration/test_federation.py +++ b/tests/integration/test_federation.py @@ -133,27 +133,6 @@ def test_elect_invalid(federation_3): assert bx[i].get_transaction(tx.id, True)[1] is None -@pytest.mark.bdb -@pytest.mark.genesis -def test_elect_disagree_prev_block(federation_3): - [bx, (s0, s1, s2)] = federation_3 - tx = input_single_create(bx[0]) - process_tx(s0) - process_tx(s1) - process_tx(s2) - process_vote(s0, True) - for i in range(3): - assert bx[i].get_transaction(tx.id, True)[1] == 'undecided' - s1.vote.last_voted_id = '5' * 64 - process_vote(s1, True) - for i in range(3): - assert bx[i].get_transaction(tx.id, True)[1] == 'undecided' - s2.vote.last_voted_id = '6' * 64 - process_vote(s2, True) - for i in range(3): - assert bx[i].get_transaction(tx.id, True)[1] is None - - @pytest.mark.bdb @pytest.mark.genesis def test_elect_sybill(federation_3): From d92f5f2024d79885bec8538549637484bf85cd94 Mon Sep 17 00:00:00 2001 From: Krish Date: Wed, 28 Jun 2017 15:35:26 +0200 Subject: [PATCH 25/42] Documentation changes (#1595) * Add documentation for `bdb-user` parameter * Remove the bdb-keyfile-passphrase parameter * Formatting changes in bdb-certs.bdb-user subsection --- .../node-config-map-and-secrets.rst | 30 +++++++++++++++++++ k8s/bigchaindb/bigchaindb-dep.yaml | 2 -- k8s/configuration/secret.yaml | 2 -- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst index 6e25c77b..1509091c 100644 --- a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst +++ b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst @@ -89,6 +89,36 @@ Note that ``ca.pem`` is just another name for ``ca.crt`` (the certificate of your BigchainDB cluster's self-signed CA). +bdb-certs.bdb-user +~~~~~~~~~~~~~~~~~~ + +This is the user name that BigchainDB uses to authenticate itself to the +backend MongoDB database. + +We need to specify the user name *as seen in the certificate* issued to +the BigchainDB instance in order to authenticate correctly. Use +the following ``openssl`` command to extract the user name from the +certificate: + +.. code:: bash + + $ openssl x509 -in \ + -inform PEM -subject -nameopt RFC2253 + +You should see an output line that resembles: + +.. code:: bash + + subject= emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE + +The ``subject`` line states the complete user name we need to use for this +field (``bdb-certs.bdb-user``), i.e. + +.. code:: bash + + emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE + + Deploy Your config-map.yaml and secret.yaml ------------------------------------------- diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 376c5c86..9aea8f7c 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -59,8 +59,6 @@ spec: value: /etc/bigchaindb/ssl/bdb-instance.pem - name: BIGCHAINDB_DATABASE_KEYFILE value: /etc/bigchaindb/ssl/bdb-instance.key - - name: BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE - value: /etc/bigchaindb/ssl/bdb-keyfile-passphrase - name: BIGCHAINDB_DATABASE_LOGIN value: /etc/bigchaindb/ssl/bdb-user # The following env var is not required for the bootstrap/first node diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml index e42ac4c9..795e4756 100644 --- a/k8s/configuration/secret.yaml +++ b/k8s/configuration/secret.yaml @@ -82,8 +82,6 @@ data: bdb-instance.pem: "" # Base64-encoded private key bdb-instance.key: "" - # Base64-encoded private key passphrase - bdb-keyfile-passphrase: "" # Base64-encoded instance authentication credentials bdb-user: "" --- From 52e097ccbaabc5854fd0063f32e37ea474de71a5 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 28 Jun 2017 14:00:42 +0200 Subject: [PATCH 26/42] Fixed typo in az acs create command --- .../template-kubernetes-azure.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/production-deployment-template/template-kubernetes-azure.rst b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst index 7027e2b3..d99596cc 100644 --- a/docs/server/source/production-deployment-template/template-kubernetes-azure.rst +++ b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst @@ -102,7 +102,7 @@ Finally, you can deploy an ACS using something like: --agent-vm-size Standard_D2_v2 \ --dns-prefix \ --ssh-key-value ~/.ssh/.pub \ - --orchestrator-type kubernetes + --orchestrator-type kubernetes \ --debug --output json From c58bd49b91247d540200e27b20f0aac6d61cd7dd Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 28 Jun 2017 14:01:48 +0200 Subject: [PATCH 27/42] clarifications in docs re config-map.yaml --- .../node-config-map-and-secrets.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst index 1509091c..2b8da6e5 100644 --- a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst +++ b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst @@ -29,8 +29,8 @@ where all data values must be base64-encoded. This is true of all Kubernetes ConfigMaps and Secrets.) -vars -~~~~ +vars.mdb-instance-name and Similar +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Your BigchainDB cluster organization should have a standard way of naming instances, so the instances in your BigchainDB node @@ -59,11 +59,14 @@ of all *other* nodes in your BigchainDB cluster * If you're deploying the first node in the cluster, the value should be ``""`` (an empty string). * If you're deploying the second node in the cluster, - the value should be one public key inside double quotes. + the value should be the BigchainDB public key of the first/original + node in the cluster, inside double quotes. For example, ``"EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"`` * If there are two or more other nodes already in the cluster, - the value should be a colon-separated list of public keys + the value should be a colon-separated list + of the BigchainDB public keys + of those other nodes, inside double quotes. For example, ``"DPjpKbmbPYPKVAuf6VSkqGCf5jzrEh69Ldef6TrLwsEQ:EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"`` From 1d6a7624f41464d73c470dc48df745e511d72260 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 28 Jun 2017 14:44:08 +0200 Subject: [PATCH 28/42] Removed un-needed trailing comma in secret.yaml --- k8s/configuration/secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml index 795e4756..7c59595b 100644 --- a/k8s/configuration/secret.yaml +++ b/k8s/configuration/secret.yaml @@ -77,7 +77,7 @@ data: # Base64-encoded CA certificate (ca.crt) ca.pem: "" # Base64-encoded CRL file - crlfile: "", + crlfile: "" # Base64-encoded BigchainDB instance certificate bdb-instance.pem: "" # Base64-encoded private key From 0feaafef2b9945017fd37c823c1a3772292e4fab Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 28 Jun 2017 16:40:52 +0200 Subject: [PATCH 29/42] entire HTTPS cert chain is needed in secret.yaml --- k8s/configuration/secret.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml index 7c59595b..cccf4788 100644 --- a/k8s/configuration/secret.yaml +++ b/k8s/configuration/secret.yaml @@ -94,8 +94,9 @@ type: Opaque data: # Base64-encoded HTTPS private key cert.key: "" - # Base64-encoded HTTPS Signed Certificate or Certificate Chain - cert.pem: "" + # Base64-encoded HTTPS certificate chain + # including all intermediate certificates and the actual certificate + cert.pem: " Date: Wed, 28 Jun 2017 17:42:32 +0200 Subject: [PATCH 30/42] Explained how to get 3scale credentials --- .../node-config-map-and-secrets.rst | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst index 2b8da6e5..8ddbff2e 100644 --- a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst +++ b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst @@ -122,6 +122,31 @@ field (``bdb-certs.bdb-user``), i.e. emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE +threescale-credentials.* +~~~~~~~~~~~~~~~~~~~~~~~~ + +You can delete the ``threescale-credentials`` Secret if you're not using 3scale. + +If you *are* using 3scale, you can get the value for ``frontend-api-dns-name`` +using something like ``echo "your.nodesubdomain.net" | base64 -w 0`` + +To get the values for ``secret-token``, ``service-id``, +``version-header`` and ``provider-key``, login to your 3scale admin, +then click **APIs** and click on **Integration** for the relevant API. +Scroll to the bottom of the page and click the small link +in the lower right corner, labelled **Download the NGINX Config files**. +You'll get a ``.zip`` file. +Unzip it, then open the ``.conf`` file and the ``.lua`` file. +You should be able to find all the values in those files. +You have to be careful because it will have values for *all* your APIs, +and some values vary from API to API. +The ``version-header`` is the timestamp in a line that looks like: + +.. code:: + + proxy_set_header X-3scale-Version "2017-06-28T14:57:34Z"; + + Deploy Your config-map.yaml and secret.yaml ------------------------------------------- From 5687735b246a89b2081b5a3569bc22c2c61e8c64 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 11:00:17 +0200 Subject: [PATCH 31/42] removed 2 instances of 'inside double quotes' --- .../node-config-map-and-secrets.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst index 8ddbff2e..dad62fc6 100644 --- a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst +++ b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst @@ -60,14 +60,13 @@ of all *other* nodes in your BigchainDB cluster the value should be ``""`` (an empty string). * If you're deploying the second node in the cluster, the value should be the BigchainDB public key of the first/original - node in the cluster, inside double quotes. + node in the cluster. For example, ``"EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"`` * If there are two or more other nodes already in the cluster, the value should be a colon-separated list of the BigchainDB public keys - of those other nodes, - inside double quotes. + of those other nodes. For example, ``"DPjpKbmbPYPKVAuf6VSkqGCf5jzrEh69Ldef6TrLwsEQ:EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"`` From 7b80bb581936476a1b955dc82d9d0d37403499c4 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 11:09:16 +0200 Subject: [PATCH 32/42] If not using 3scale, they can also leave vals blank --- .../node-config-map-and-secrets.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst index dad62fc6..7bcbb28d 100644 --- a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst +++ b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst @@ -124,7 +124,9 @@ field (``bdb-certs.bdb-user``), i.e. threescale-credentials.* ~~~~~~~~~~~~~~~~~~~~~~~~ -You can delete the ``threescale-credentials`` Secret if you're not using 3scale. +If you're not using 3scale, +you can delete the ``threescale-credentials`` Secret +or leave all the values blank (``""``). If you *are* using 3scale, you can get the value for ``frontend-api-dns-name`` using something like ``echo "your.nodesubdomain.net" | base64 -w 0`` From 54e0fc4f57492e66830e67600ceeb5dc5fa77706 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 11:28:51 +0200 Subject: [PATCH 33/42] Clarify the required SSL cert order in cert chain --- k8s/configuration/secret.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml index cccf4788..f9c4aeb5 100644 --- a/k8s/configuration/secret.yaml +++ b/k8s/configuration/secret.yaml @@ -95,7 +95,9 @@ data: # Base64-encoded HTTPS private key cert.key: "" # Base64-encoded HTTPS certificate chain - # including all intermediate certificates and the actual certificate + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert if from DigiCert, download "Best format for nginx". cert.pem: " Date: Thu, 29 Jun 2017 14:19:08 +0200 Subject: [PATCH 34/42] Document graphite configuration parameter (#1594) * document graphite configuration parameter * add example * graphite settings docs follows wsserver * Removed outer JSON braces @ graphite.host docs --- .../source/server-reference/configuration.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 76a76456..7edc184c 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -45,6 +45,7 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_DATABASE_KEYFILE`
`BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE`
`BIGCHAINDB_DATABASE_CRLFILE`
+`BIGCHAINDB_GRAPHITE_HOST`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -497,3 +498,29 @@ logging of the `core.py` module to be more verbose, you would set the ``` **Defaults to**: `"{}"` + + +## graphite.host + +The host name or IP address of a server listening for statsd events on UDP +port 8125. This defaults to `localhost`, and if no statsd collector is running, +the events are simply dropped by the operating system. + +**Example using environment variables** +```text +export BIGCHAINDB_GRAPHITE_HOST=10.0.0.5 +``` + +**Example config file snippet** +```js +"graphite": { + "host": "10.0.0.5" +} +``` + +**Default values (from a config file)** +```js +"graphite": { + "host": "localhost" +} +``` From d3924213ee4f74be7f496065a863fc25c99826e1 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 14:35:23 +0200 Subject: [PATCH 35/42] edits in nginx-3scale service docs --- .../node-on-kubernetes.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index fb4219f1..92c7c424 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -138,14 +138,17 @@ Step 4.1: Vanilla NGINX Step 4.2: OpenResty NGINX + 3scale ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - * This configuration is located in the file ``nginx/nginx-3scale-svc.yaml``. * You have to enable HTTPS for this one and will need an HTTPS certificate - for your domain + for your domain. - * You should have already created the Kubernetes Secret in the previous - step. + * You should have already created the necessary Kubernetes Secrets in the previous + step (e.g. ``https-certs`` and ``threescale-credentials``). + + * This configuration is located in the file ``nginx-3scale/nginx-3scale-svc.yaml``. + + * Set the ``metadata.name`` and ``metadata.labels.name`` to the value + set in ``ngx-instance-name`` in the ConfigMap above. * Set the ``spec.selector.app`` to the value set in ``ngx-instance-name`` in the ConfigMap followed by ``-dep``. For example, if the value set in the From a72bf56089b794dbd1e360aa8ed95844fb7366af Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 15:09:27 +0200 Subject: [PATCH 36/42] copyedited docs re assigning DNS name to NGINX public IP --- .../node-on-kubernetes.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index 92c7c424..0310a4df 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -170,20 +170,18 @@ Step 5: Assign DNS Name to the NGINX Public IP `_ or are using HTTPS certificates tied to a domain. - * The following command can help you find out if the nginx service started + * The following command can help you find out if the NGINX service started above has been assigned a public IP or external IP address: .. code:: bash $ kubectl --context k8s-bdb-test-cluster-0 get svc -w - * Once a public IP is assigned, you can log in to the Azure portal and map it to + * Once a public IP is assigned, you can map it to a DNS name. - - * We usually assign ``bdb-test-cluster-0``, ``bdb-test-cluster-1`` and + We usually assign ``bdb-test-cluster-0``, ``bdb-test-cluster-1`` and so on in our documentation. - - * Let us assume that we assigned the unique name of ``bdb-test-cluster-0`` here. + Let's assume that we assign the unique name of ``bdb-test-cluster-0`` here. **Set up DNS mapping in Azure.** @@ -198,7 +196,7 @@ have the Azure DNS prefix name along with a long random string, without the (for example, ``bdb-test-cluster-0``), click ``Save``, and wait for the changes to be applied. -To verify the DNS setting is operational, you can run ``nslookup `` from your local Linux shell. This will ensure that when you scale the replica set later, other MongoDB From 1034db1ce50ede3ced81133018c1ca48e2d1baa4 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 15:32:08 +0200 Subject: [PATCH 37/42] Fixed name of https-certs volume mount in nginx-3scale-dep.yaml --- k8s/nginx-3scale/nginx-3scale-dep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/nginx-3scale/nginx-3scale-dep.yaml b/k8s/nginx-3scale/nginx-3scale-dep.yaml index 1dacf617..7951e14d 100644 --- a/k8s/nginx-3scale/nginx-3scale-dep.yaml +++ b/k8s/nginx-3scale/nginx-3scale-dep.yaml @@ -84,7 +84,7 @@ spec: timeoutSeconds: 10 restartPolicy: Always volumes: - - name: https + - name: https-certs secret: secretName: https-certs defaultMode: 0400 From 92ec8f613e67d4e670a3d26dc9b9ea6d9205306f Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 16:02:34 +0200 Subject: [PATCH 38/42] Fixed spelling & grammar stuff in docs re MDB StatefulSet --- .../production-deployment-template/node-on-kubernetes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index 0310a4df..34fbfda4 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -453,11 +453,11 @@ Step 11: Start a Kubernetes StatefulSet for MongoDB * Note how the MongoDB container uses the ``mongo-db-claim`` and the ``mongo-configdb-claim`` PersistentVolumeClaims for its ``/data/db`` and - ``/data/configdb`` diretories (mount path). + ``/data/configdb`` directories (mount paths). * Note also that we use the pod's ``securityContext.capabilities.add`` specification to add the ``FOWNER`` capability to the container. That is - because MongoDB container has the user ``mongodb``, with uid ``999`` and + because the MongoDB container has the user ``mongodb``, with uid ``999`` and group ``mongodb``, with gid ``999``. When this container runs on a host with a mounted disk, the writes fail when there is no user with uid ``999``. To avoid this, we use the Docker From 6b6bfe173331c862d167d5d7e8917521edcbad3c Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 16:39:01 +0200 Subject: [PATCH 39/42] Explained how to log in to the MongoDB pod --- .../node-on-kubernetes.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index 34fbfda4..7d0a8f83 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -491,12 +491,23 @@ Step 11: Start a Kubernetes StatefulSet for MongoDB Step 12: Configure Users and Access Control for MongoDB ------------------------------------------------------- - * Create a user on MongoDB with authorization to create more users and assign + * In this step, you will create a user on MongoDB with authorization + to create more users and assign roles to them. Note: You need to do this only when setting up the first MongoDB node of the cluster. - Log in to the MongoDB instance and open a mongo shell using the certificates + * Find out the name of your MongoDB pod by reading the output + of the ``kubectl ... get pods`` command at the end of the last step. + It should be something like ``mdb-instance-0-ss-0``. + + * Log in to the MongoDB pod using: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 exec -it bash + + * Open a mongo shell using the certificates already present at ``/etc/mongod/ssl/`` .. code:: bash From 69cdfd56cfa6ed8dd9d1de2e785e24731edcfc08 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 29 Jun 2017 17:02:23 +0200 Subject: [PATCH 40/42] Added note about what to expect from MongoDB's db.auth() command --- .../production-deployment-template/node-on-kubernetes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index 7d0a8f83..4237fbe3 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -561,6 +561,9 @@ Step 12: Configure Users and Access Control for MongoDB PRIMARY> use admin PRIMARY> db.auth("adminUser", "superstrongpassword") + ``db.auth()`` returns 0 when authentication is not successful, + and 1 when successful. + * We need to specify the user name *as seen in the certificate* issued to the BigchainDB instance in order to authenticate correctly. Use the following ``openssl`` command to extract the user name from the From b1ad6045b6cce051fe9b6eb84cf9706e1bb6d4f8 Mon Sep 17 00:00:00 2001 From: libscott Date: Fri, 30 Jun 2017 09:40:50 +0200 Subject: [PATCH 41/42] Rename "output" to "output_index" in /api/v1/outputs (#1596) * rename output to output_index in /api/v1/outputs * rename transaction.input.fulfills.output to output_index * fix rethinkdb queries for output_index --- bigchaindb/backend/mongodb/query.py | 4 ++-- bigchaindb/backend/mongodb/schema.py | 2 +- bigchaindb/backend/rethinkdb/query.py | 4 ++-- bigchaindb/backend/rethinkdb/schema.py | 4 ++-- bigchaindb/common/schema/transaction.yaml | 4 ++-- bigchaindb/common/transaction.py | 4 ++-- bigchaindb/web/views/outputs.py | 2 +- docs/server/source/data-models/inputs-outputs.rst | 6 +++--- docs/server/source/http-client-server-api.rst | 8 ++++---- tests/common/test_transaction.py | 10 +++++----- tests/web/test_outputs.py | 8 ++++---- 11 files changed, 28 insertions(+), 28 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index a1e97b64..673f643f 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -158,7 +158,7 @@ def get_spent(conn, transaction_id, output): 'block.transactions.inputs': { '$elemMatch': { 'fulfills.transaction_id': transaction_id, - 'fulfills.output': output, + 'fulfills.output_index': output, }, }, }}, @@ -167,7 +167,7 @@ def get_spent(conn, transaction_id, output): 'block.transactions.inputs': { '$elemMatch': { 'fulfills.transaction_id': transaction_id, - 'fulfills.output': output, + 'fulfills.output_index': output, }, }, }}, diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index c9c5bbb4..e398560f 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -72,7 +72,7 @@ def create_bigchain_secondary_index(conn, dbname): conn.conn[dbname]['bigchain']\ .create_index([ ('block.transactions.inputs.fulfills.transaction_id', ASCENDING), - ('block.transactions.inputs.fulfills.output', ASCENDING), + ('block.transactions.inputs.fulfills.output_index', ASCENDING), ], name='inputs') diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index a2e97d9a..cac9cc94 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -123,7 +123,7 @@ def get_spent(connection, transaction_id, output): .concat_map(lambda doc: doc['block']['transactions']) .filter(lambda transaction: transaction['inputs'].contains( lambda input_: input_['fulfills'] == { - 'transaction_id': transaction_id, 'output': output}))) + 'transaction_id': transaction_id, 'output_index': output}))) @register_query(RethinkDBConnection) @@ -287,7 +287,7 @@ def unwind_block_transactions(block): def get_spending_transactions(connection, links): query = ( r.table('bigchain') - .get_all(*[(l['transaction_id'], l['output']) for l in links], + .get_all(*[(l['transaction_id'], l['output_index']) for l in links], index='inputs') .concat_map(unwind_block_transactions) # filter transactions spending output diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index f96e8a86..ea6f4e25 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -86,9 +86,9 @@ def create_bigchain_secondary_index(connection, dbname): .index_create('inputs', r.row['block']['transactions'] .concat_map(lambda tx: tx['inputs']['fulfills']) - .with_fields('transaction_id', 'output') + .with_fields('transaction_id', 'output_index') .map(lambda fulfills: [fulfills['transaction_id'], - fulfills['output']]), + fulfills['output_index']]), multi=True)) # wait for rethinkdb to finish creating secondary indexes diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index 633579a5..f393d067 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -203,10 +203,10 @@ definitions: Reference to the output that is being spent. additionalProperties: false required: - - output + - output_index - transaction_id properties: - output: + output_index: "$ref": "#/definitions/offset" description: | Index of the output containing the condition being fulfilled diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index f3cf476d..9b44ede6 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -178,7 +178,7 @@ class TransactionLink(object): :class:`~bigchaindb.common.transaction.TransactionLink` """ try: - return cls(link['transaction_id'], link['output']) + return cls(link['transaction_id'], link['output_index']) except TypeError: return cls() @@ -193,7 +193,7 @@ class TransactionLink(object): else: return { 'transaction_id': self.txid, - 'output': self.output, + 'output_index': self.output, } def to_uri(self, path=''): diff --git a/bigchaindb/web/views/outputs.py b/bigchaindb/web/views/outputs.py index 253ae0bd..2f63c07f 100644 --- a/bigchaindb/web/views/outputs.py +++ b/bigchaindb/web/views/outputs.py @@ -22,5 +22,5 @@ class OutputListApi(Resource): with pool() as bigchain: outputs = bigchain.get_outputs_filtered(args['public_key'], args['spent']) - return [{'transaction_id': output.txid, 'output': output.output} + return [{'transaction_id': output.txid, 'output_index': output.output} for output in outputs] diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index ea9a1dcd..62deb9d3 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -132,7 +132,7 @@ If there is only one *current owner*, the fulfillment will be a simple signature "owners_before": [""], "fulfillment": "cf:4:RxFzIE679tFBk8zwEgizhmTuciAylvTUwy6EL6ehddHFJOhK5F4IjwQ1xLu2oQK9iyRCZJdfWAefZVjTt3DeG5j2exqxpGliOPYseNkRAWEakqJ_UrCwgnj92dnFRAEE", "fulfills": { - "output": 0, + "output_index": 0, "transaction_id": "11b3e7d893cc5fdfcf1a1706809c7def290a3b10b0bef6525d10b024649c42d3" } } @@ -151,7 +151,7 @@ If there are multiple *current owners*, the fulfillment will be a little differe "owners_before": ["",""], "fulfillment": "cf:2:AQIBAgEBYwAEYEv6O5HjHGl7OWo2Tu5mWcWQcL_OGrFuUjyej-dK3LM99TbZsRd8c9luQhU30xCH5AdNaupxg-pLHuk8DoSaDA1MHQGXUZ80a_cV-4UaaaCpdey8K0CEcJxre0X96hTHCwABAWMABGBnsuHExhuSj5Mdm-q0KoPgX4nAt0s00k1WTMCzuUpQIp6aStLoTSMlsvS4fmDtOSv9gubekKLuHTMAk-LQFSKF1JdzwaVWAA2UOv0v_OS2gY3A-r0kRq8HtzjYdcmVswUA", "fulfills": { - "output": 0, + "output_index": 0, "transaction_id": "e4805f1bfc999d6409b38e3a4c3b2fafad7c1280eb0d441da7083e945dd89eb8" } } @@ -160,5 +160,5 @@ If there are multiple *current owners*, the fulfillment will be a little differe - ``owners_before``: A list of public keys of the owners before the transaction; in this case it has two owners, hence two public keys. - ``fulfillment``: A crypto-conditions URI that encodes the cryptographic fulfillments like signatures and others;'cf' indicates this is a fulfillment, '2' indicates the condition type is THRESHOLD-SHA-256 (while '4' in `One Current Owner`_ indicates its condition type is ED25519). - ``fulfills``: Pointer to an output from a previous transaction that is being spent - - ``output``: The index of the output in a previous transaction + - ``output_index``: The index of the output in a previous transaction - ``transaction_id``: ID of the transaction diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index c7a2d3e3..618903a8 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -206,11 +206,11 @@ unspent outputs. [ { - "output": 0, + "output_index": 0, "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" }, { - "output": 1, + "output_index": 1, "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" } ] @@ -238,7 +238,7 @@ unspent outputs. [ { - "output": 0, + "output_index": 0, "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" } ] @@ -266,7 +266,7 @@ unspent outputs. [ { - "output": 1, + "output_index": 1, "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" } ] diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 368789bb..2cdfaac1 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -366,7 +366,7 @@ def test_transaction_link_serialization(): tx_id = 'a transaction id' expected = { 'transaction_id': tx_id, - 'output': 0, + 'output_index': 0, } tx_link = TransactionLink(tx_id, 0) @@ -389,7 +389,7 @@ def test_transaction_link_deserialization(): expected = TransactionLink(tx_id, 0) tx_link = { 'transaction_id': tx_id, - 'output': 0, + 'output_index': 0, } tx_link = TransactionLink.from_dict(tx_link) @@ -842,7 +842,7 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, 'fulfillment': None, 'fulfills': { 'transaction_id': tx.id, - 'output': 0 + 'output_index': 0 } } ], @@ -891,7 +891,7 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, 'fulfillment': None, 'fulfills': { 'transaction_id': tx.id, - 'output': 0 + 'output_index': 0 } }, { 'owners_before': [ @@ -900,7 +900,7 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, 'fulfillment': None, 'fulfills': { 'transaction_id': tx.id, - 'output': 1 + 'output_index': 1 } } ], diff --git a/tests/web/test_outputs.py b/tests/web/test_outputs.py index d20068d0..8ef90b73 100644 --- a/tests/web/test_outputs.py +++ b/tests/web/test_outputs.py @@ -14,8 +14,8 @@ def test_get_outputs_endpoint(client, user_pk): gof.return_value = [m, m] res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk)) assert res.json == [ - {'transaction_id': 'a', 'output': 0}, - {'transaction_id': 'a', 'output': 0} + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 0} ] assert res.status_code == 200 gof.assert_called_once_with(user_pk, None) @@ -29,7 +29,7 @@ def test_get_outputs_endpoint_unspent(client, user_pk): gof.return_value = [m] params = '?spent=False&public_key={}'.format(user_pk) res = client.get(OUTPUTS_ENDPOINT + params) - assert res.json == [{'transaction_id': 'a', 'output': 0}] + assert res.json == [{'transaction_id': 'a', 'output_index': 0}] assert res.status_code == 200 gof.assert_called_once_with(user_pk, False) @@ -42,7 +42,7 @@ def test_get_outputs_endpoint_spent(client, user_pk): gof.return_value = [m] params = '?spent=true&public_key={}'.format(user_pk) res = client.get(OUTPUTS_ENDPOINT + params) - assert res.json == [{'transaction_id': 'a', 'output': 0}] + assert res.json == [{'transaction_id': 'a', 'output_index': 0}] assert res.status_code == 200 gof.assert_called_once_with(user_pk, True) From 5e9b7f4ffe503d66131cbd96266cb70666a006b9 Mon Sep 17 00:00:00 2001 From: libscott Date: Fri, 30 Jun 2017 09:44:22 +0200 Subject: [PATCH 42/42] Remove condition details signature, rename subfulfillments to subconditions (#1589) * implement output.condition.details marshalling and remove signature field * rename outputs[].condition.details.subfulfillments to subconditions * simpler threshold depth overlow handling * pass public_key as kwarg * change ccv1 condition uri in docs * import base58 at top in test_transaction --- bigchaindb/common/exceptions.py | 4 + bigchaindb/common/schema/transaction.yaml | 61 ++++++++++------ bigchaindb/common/transaction.py | 73 +++++++++++++++---- bigchaindb/utils.py | 4 +- .../source/data-models/inputs-outputs.rst | 29 +++----- tests/assets/test_divisible_assets.py | 71 +++++++++--------- tests/common/test_transaction.py | 12 ++- .../validation/test_transaction_structure.py | 35 ++++++++- 8 files changed, 195 insertions(+), 94 deletions(-) diff --git a/bigchaindb/common/exceptions.py b/bigchaindb/common/exceptions.py index 258001b8..ec4c9702 100644 --- a/bigchaindb/common/exceptions.py +++ b/bigchaindb/common/exceptions.py @@ -106,3 +106,7 @@ class SybilError(ValidationError): class DuplicateTransaction(ValidationError): """Raised if a duplicated transaction is found""" + + +class ThresholdTooDeep(ValidationError): + """Raised if threshold condition is too deep""" diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index f393d067..72b51cf1 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -150,8 +150,7 @@ definitions: - uri properties: details: - type: object - additionalProperties: true + "$ref": "#/definitions/condition_details" uri: type: string pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})?(.+)$" @@ -174,28 +173,14 @@ definitions: description: | List of public keys of the previous owners of the asset. fulfillment: + description: | + Fulfillment of an `Output.condition`_, or, put a different way, a payload + that satisfies the condition of a previous output to prove that the + creator(s) of this transaction have control over the listed asset. anyOf: - - type: object - additionalProperties: false - properties: - bitmask: - type: integer - public_key: - type: string - type: - type: string - signature: - anyOf: - - type: string - - type: 'null' - type_id: - type: integer - description: | - Fulfillment of an `Output.condition`_, or, put a different way, a payload - that satisfies the condition of a previous output to prove that the - creator(s) of this transaction have control over the listed asset. - type: string pattern: "^[a-zA-Z0-9_-]*$" + - "$ref": "#/definitions/condition_details" fulfills: anyOf: - type: 'object' @@ -224,3 +209,37 @@ definitions: additionalProperties: true minProperties: 1 - type: 'null' + condition_details: + description: | + Details needed to reconstruct the condition associated with an output. + Currently, BigchainDB only supports ed25519 and threshold condition types. + anyOf: + - type: object + additionalProperties: false + required: + - type + - public_key + properties: + type: + type: string + pattern: "^ed25519-sha-256$" + public_key: + "$ref": "#/definitions/base58" + - type: object + additionalProperties: false + required: + - type + - threshold + - subconditions + properties: + type: + type: "string" + pattern: "^threshold-sha-256$" + threshold: + type: integer + minimum: 1 + maximum: 100 + subconditions: + type: array + items: + "$ref": "#/definitions/condition_details" diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 9b44ede6..a377b994 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -4,12 +4,13 @@ from functools import reduce import base58 from cryptoconditions import Fulfillment, ThresholdSha256, Ed25519Sha256 from cryptoconditions.exceptions import ( - ParsingError, ASN1DecodeError, ASN1EncodeError) + ParsingError, ASN1DecodeError, ASN1EncodeError, UnsupportedTypeError) from bigchaindb.common.crypto import PrivateKey, hash_data from bigchaindb.common.exceptions import (KeypairMismatchException, InvalidHash, InvalidSignature, - AmountError, AssetIdMismatch) + AmountError, AssetIdMismatch, + ThresholdTooDeep) from bigchaindb.common.utils import serialize @@ -66,15 +67,7 @@ class Input(object): try: fulfillment = self.fulfillment.serialize_uri() except (TypeError, AttributeError, ASN1EncodeError): - # NOTE: When a non-signed transaction is casted to a dict, - # `self.inputs` value is lost, as in the node's - # transaction model that is saved to the database, does not - # account for its dictionary form but just for its signed uri - # form. - # Hence, when a non-signed fulfillment is to be cast to a - # dict, we just call its internal `to_dict` method here and - # its `from_dict` method in `Fulfillment.from_dict`. - fulfillment = self.fulfillment.to_dict() + fulfillment = _fulfillment_to_details(self.fulfillment) try: # NOTE: `self.fulfills` can be `None` and that's fine @@ -125,11 +118,63 @@ class Input(object): except TypeError: # NOTE: See comment about this special case in # `Input.to_dict` - fulfillment = Fulfillment.from_dict(data['fulfillment']) + fulfillment = _fulfillment_from_details(data['fulfillment']) fulfills = TransactionLink.from_dict(data['fulfills']) return cls(fulfillment, data['owners_before'], fulfills) +def _fulfillment_to_details(fulfillment): + """ + Encode a fulfillment as a details dictionary + + Args: + fulfillment: Crypto-conditions Fulfillment object + """ + + if fulfillment.type_name == 'ed25519-sha-256': + return { + 'type': 'ed25519-sha-256', + 'public_key': base58.b58encode(fulfillment.public_key), + } + + if fulfillment.type_name == 'threshold-sha-256': + subconditions = [ + _fulfillment_to_details(cond['body']) + for cond in fulfillment.subconditions + ] + return { + 'type': 'threshold-sha-256', + 'threshold': fulfillment.threshold, + 'subconditions': subconditions, + } + + raise UnsupportedTypeError(fulfillment.type_name) + + +def _fulfillment_from_details(data): + """ + Load a fulfillment for a signing spec dictionary + + Args: + data: tx.output[].condition.details dictionary + """ + if data['type'] == 'ed25519-sha-256': + public_key = base58.b58decode(data['public_key']) + return Ed25519Sha256(public_key=public_key) + + if data['type'] == 'threshold-sha-256': + try: + threshold = ThresholdSha256(data['threshold']) + for cond in data['subconditions']: + cond = _fulfillment_from_details(cond) + threshold.add_subfulfillment(cond) + return threshold + except RecursionError: + raise ThresholdTooDeep() + + raise UnsupportedTypeError(data.get('type')) + + class TransactionLink(object): """An object for unidirectional linking to a Transaction's Output. @@ -262,7 +307,7 @@ class Output(object): # and fulfillment! condition = {} try: - condition['details'] = self.fulfillment.to_dict() + condition['details'] = _fulfillment_to_details(self.fulfillment) except AttributeError: pass @@ -389,7 +434,7 @@ class Output(object): :class:`~bigchaindb.common.transaction.Output` """ try: - fulfillment = Fulfillment.from_dict(data['condition']['details']) + fulfillment = _fulfillment_from_details(data['condition']['details']) except KeyError: # NOTE: Hashlock condition case fulfillment = data['condition']['uri'] diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index f87916b7..1738d70b 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -96,8 +96,8 @@ def condition_details_has_owner(condition_details, owner): bool: True if the public key is found in the condition details, False otherwise """ - if 'subfulfillments' in condition_details: - result = condition_details_has_owner(condition_details['subfulfillments'], owner) + if 'subconditions' in condition_details: + result = condition_details_has_owner(condition_details['subconditions'], owner) if result: return True diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index 62deb9d3..da190477 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -81,30 +81,20 @@ to spend the asset. For example: { "condition": { "details": { - "bitmask": 41, - "subfulfillments": [ + "type": "threshold-sha-256", + "threshold": 2, + "subconditions": [ { - "bitmask": 32, "public_key": "", - "signature": null, - "type": "fulfillment", - "type_id": 4, - "weight": 1 + "type": "ed25519-sha-256", }, { - "bitmask": 32, "public_key": "", - "signature": null, - "type": "fulfillment", - "type_id": 4, - "weight": 1 + "type": "ed25519-sha-256", } ], - "threshold": 2, - "type": "fulfillment", - "type_id": 2 }, - "uri": "cc:2:29:ytNK3X6-bZsbF-nCGDTuopUIMi1HCyCkyPewm6oLI3o:206"}, + "uri": "ni:///sha-256;PNYwdxaRaNw60N6LDFzOWO97b8tJeragczakL8PrAPc?fpt=ed25519-sha-256&cost=131072"}, "public_keys": [ "", "" @@ -112,11 +102,10 @@ to spend the asset. For example: } -- ``subfulfillments``: a list of fulfillments - - ``weight``: integer weight for each subfulfillment's contribution to the threshold -- ``threshold``: threshold to reach for the subfulfillments to reach a valid fulfillment +- ``subconditions``: a list of condition specs +- ``threshold``: threshold to reach for the subconditions to reach a valid fulfillment -The ``weight``s and ``threshold`` could be adjusted. For example, if the ``threshold`` was changed to 1 above, then only one of the new owners would have to provide a signature to spend the asset. +The ``threshold`` can be adjusted. For example, if the ``threshold`` was changed to 1 above, then only one of the new owners would have to provide a signature to spend the asset. If it is desired to give a different weight to a subcondition, it should be specified multiple times. Inputs ------ diff --git a/tests/assets/test_divisible_assets.py b/tests/assets/test_divisible_assets.py index e1ea726f..123fe063 100644 --- a/tests/assets/test_divisible_assets.py +++ b/tests/assets/test_divisible_assets.py @@ -52,8 +52,8 @@ def test_single_in_single_own_single_out_multiple_own_create(b, user_pk): assert tx_signed.outputs[0].amount == 100 output = tx_signed.outputs[0].to_dict() - assert 'subfulfillments' in output['condition']['details'] - assert len(output['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in output['condition']['details'] + assert len(output['condition']['details']['subconditions']) == 2 assert len(tx_signed.inputs) == 1 @@ -76,8 +76,8 @@ def test_single_in_single_own_multiple_out_mix_own_create(b, user_pk): assert tx_signed.outputs[1].amount == 50 output_cid1 = tx_signed.outputs[1].to_dict() - assert 'subfulfillments' in output_cid1['condition']['details'] - assert len(output_cid1['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in output_cid1['condition']['details'] + assert len(output_cid1['condition']['details']['subconditions']) == 2 assert len(tx_signed.inputs) == 1 @@ -89,6 +89,7 @@ def test_single_in_single_own_multiple_out_mix_own_create(b, user_pk): def test_single_in_multiple_own_single_out_single_own_create(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details tx = Transaction.create([b.me, user_pk], [([user_pk], 100)]) tx_signed = tx.sign([b.me_private, user_sk]) @@ -97,9 +98,9 @@ def test_single_in_multiple_own_single_out_single_own_create(b, user_pk, assert tx_signed.outputs[0].amount == 100 assert len(tx_signed.inputs) == 1 - ffill = tx_signed.inputs[0].fulfillment.to_dict() - assert 'subfulfillments' in ffill - assert len(ffill['subfulfillments']) == 2 + ffill = _fulfillment_to_details(tx_signed.inputs[0].fulfillment) + assert 'subconditions' in ffill + assert len(ffill['subconditions']) == 2 # TRANSFER divisible asset @@ -207,8 +208,8 @@ def test_single_in_single_own_single_out_multiple_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 condition = tx_transfer_signed.outputs[0].to_dict() - assert 'subfulfillments' in condition['condition']['details'] - assert len(condition['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in condition['condition']['details'] + assert len(condition['condition']['details']['subconditions']) == 2 assert len(tx_transfer_signed.inputs) == 1 @@ -248,8 +249,8 @@ def test_single_in_single_own_multiple_out_mix_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[1].amount == 50 output_cid1 = tx_transfer_signed.outputs[1].to_dict() - assert 'subfulfillments' in output_cid1['condition']['details'] - assert len(output_cid1['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in output_cid1['condition']['details'] + assert len(output_cid1['condition']['details']['subconditions']) == 2 assert len(tx_transfer_signed.inputs) == 1 @@ -264,6 +265,7 @@ def test_single_in_single_own_multiple_out_mix_own_transfer(b, user_pk, def test_single_in_multiple_own_single_out_single_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([b.me, user_pk], 100)]) @@ -286,9 +288,9 @@ def test_single_in_multiple_own_single_out_single_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 assert len(tx_transfer_signed.inputs) == 1 - ffill = tx_transfer_signed.inputs[0].fulfillment.to_dict() - assert 'subfulfillments' in ffill - assert len(ffill['subfulfillments']) == 2 + ffill = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + assert 'subconditions' in ffill + assert len(ffill['subconditions']) == 2 # TRANSFER divisible asset @@ -334,6 +336,7 @@ def test_multiple_in_single_own_single_out_single_own_transfer(b, user_pk, def test_multiple_in_multiple_own_single_out_single_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([user_pk, b.me], 50), ([user_pk, b.me], 50)]) @@ -356,12 +359,12 @@ def test_multiple_in_multiple_own_single_out_single_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 assert len(tx_transfer_signed.inputs) == 2 - ffill_fid0 = tx_transfer_signed.inputs[0].fulfillment.to_dict() - ffill_fid1 = tx_transfer_signed.inputs[1].fulfillment.to_dict() - assert 'subfulfillments' in ffill_fid0 - assert 'subfulfillments' in ffill_fid1 - assert len(ffill_fid0['subfulfillments']) == 2 - assert len(ffill_fid1['subfulfillments']) == 2 + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid0['subconditions']) == 2 + assert len(ffill_fid1['subconditions']) == 2 # TRANSFER divisible asset @@ -375,6 +378,7 @@ def test_multiple_in_multiple_own_single_out_single_own_transfer(b, user_pk, def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([user_pk], 50), ([user_pk, b.me], 50)]) @@ -397,11 +401,11 @@ def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 assert len(tx_transfer_signed.inputs) == 2 - ffill_fid0 = tx_transfer_signed.inputs[0].fulfillment.to_dict() - ffill_fid1 = tx_transfer_signed.inputs[1].fulfillment.to_dict() - assert 'subfulfillments' not in ffill_fid0 - assert 'subfulfillments' in ffill_fid1 - assert len(ffill_fid1['subfulfillments']) == 2 + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' not in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid1['subconditions']) == 2 # TRANSFER divisible asset @@ -416,6 +420,7 @@ def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(b, user_pk, def test_muiltiple_in_mix_own_multiple_out_mix_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([user_pk], 50), ([user_pk, b.me], 50)]) @@ -442,15 +447,15 @@ def test_muiltiple_in_mix_own_multiple_out_mix_own_transfer(b, user_pk, cond_cid0 = tx_transfer_signed.outputs[0].to_dict() cond_cid1 = tx_transfer_signed.outputs[1].to_dict() - assert 'subfulfillments' not in cond_cid0['condition']['details'] - assert 'subfulfillments' in cond_cid1['condition']['details'] - assert len(cond_cid1['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' not in cond_cid0['condition']['details'] + assert 'subconditions' in cond_cid1['condition']['details'] + assert len(cond_cid1['condition']['details']['subconditions']) == 2 - ffill_fid0 = tx_transfer_signed.inputs[0].fulfillment.to_dict() - ffill_fid1 = tx_transfer_signed.inputs[1].fulfillment.to_dict() - assert 'subfulfillments' not in ffill_fid0 - assert 'subfulfillments' in ffill_fid1 - assert len(ffill_fid1['subfulfillments']) == 2 + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' not in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid1['subconditions']) == 2 # TRANSFER divisible asset diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 2cdfaac1..ffe42d1e 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -4,7 +4,7 @@ Tests for transaction validation are separate. """ from copy import deepcopy -from base58 import b58decode +from base58 import b58encode, b58decode from pytest import raises @@ -82,7 +82,10 @@ def test_output_serialization(user_Ed25519, user_pub): expected = { 'condition': { 'uri': user_Ed25519.condition_uri, - 'details': user_Ed25519.to_dict(), + 'details': { + 'type': 'ed25519-sha-256', + 'public_key': b58encode(user_Ed25519.public_key), + }, }, 'public_keys': [user_pub], 'amount': '1', @@ -100,7 +103,10 @@ def test_output_deserialization(user_Ed25519, user_pub): cond = { 'condition': { 'uri': user_Ed25519.condition_uri, - 'details': user_Ed25519.to_dict() + 'details': { + 'type': 'ed25519-sha-256', + 'public_key': b58encode(user_Ed25519.public_key), + }, }, 'public_keys': [user_pub], 'amount': '1', diff --git a/tests/validation/test_transaction_structure.py b/tests/validation/test_transaction_structure.py index 2a7d92fb..22c09651 100644 --- a/tests/validation/test_transaction_structure.py +++ b/tests/validation/test_transaction_structure.py @@ -5,9 +5,11 @@ structural / schematic issues are caught when reading a transaction """ import pytest +from unittest.mock import MagicMock from bigchaindb.common.exceptions import (AmountError, InvalidHash, - SchemaValidationError) + SchemaValidationError, + ThresholdTooDeep) from bigchaindb.models import Transaction @@ -161,6 +163,37 @@ def test_high_amounts(create_tx): validate(create_tx) +################################################################################ +# Conditions + +def test_handle_threshold_overflow(): + from bigchaindb.common import transaction + + cond = { + 'type': 'ed25519-sha-256', + 'public_key': 'a' * 43, + } + for i in range(1000): + cond = { + 'type': 'threshold-sha-256', + 'threshold': 1, + 'subconditions': [cond], + } + with pytest.raises(ThresholdTooDeep): + transaction._fulfillment_from_details(cond) + + +def test_unsupported_condition_type(): + from bigchaindb.common import transaction + from cryptoconditions.exceptions import UnsupportedTypeError + + with pytest.raises(UnsupportedTypeError): + transaction._fulfillment_from_details({'type': 'a'}) + + with pytest.raises(UnsupportedTypeError): + transaction._fulfillment_to_details(MagicMock(type_name='a')) + + ################################################################################ # Version