diff --git a/.ci/travis-before-script.sh b/.ci/travis-before-script.sh index dc4f0a09..3b638ead 100755 --- a/.ci/travis-before-script.sh +++ b/.ci/travis-before-script.sh @@ -4,9 +4,31 @@ set -e -x if [[ "${TOXENV}" == *-rdb ]]; then rethinkdb --daemon -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb ]]; then - wget http://downloads.mongodb.org/linux/mongodb-linux-x86_64-3.4.1.tgz -O /tmp/mongodb.tgz +elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ + -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then + # Connect to MongoDB on port 27017 via a normal, unsecure connection if + # BIGCHAINDB_DATABASE_SSL is unset. + # It is unset in this case in .travis.yml. + wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1404-3.4.4.tgz -O /tmp/mongodb.tgz tar -xvf /tmp/mongodb.tgz mkdir /tmp/mongodb-data - ${PWD}/mongodb-linux-x86_64-3.4.1/bin/mongod --dbpath=/tmp/mongodb-data --replSet=bigchain-rs &> /dev/null & + ${PWD}/mongodb-linux-x86_64-ubuntu1404-3.4.4/bin/mongod \ + --dbpath=/tmp/mongodb-data --replSet=bigchain-rs &> /dev/null & +elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ + "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then + # Connect to MongoDB on port 27017 via TLS/SSL connection if + # BIGCHAINDB_DATABASE_SSL is set. + # It is set to 'true' here in .travis.yml. Dummy certificates for testing + # are stored under bigchaindb/tests/backend/mongodb-ssl/certs/ directory. + wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1404-3.4.4.tgz -O /tmp/mongodb-ssl.tgz + tar -xvf /tmp/mongodb-ssl.tgz + mkdir /tmp/mongodb-ssl-data + ${PWD}/mongodb-linux-x86_64-ubuntu1404-3.4.4/bin/mongod \ + --dbpath=/tmp/mongodb-ssl-data \ + --replSet=bigchain-rs \ + --sslAllowInvalidHostnames \ + --sslMode=requireSSL \ + --sslCAFile=$TRAVIS_BUILD_DIR/tests/backend/mongodb-ssl/certs/ca.crt \ + --sslCRLFile=$TRAVIS_BUILD_DIR/tests/backend/mongodb-ssl/certs/crl.pem \ + --sslPEMKeyFile=$TRAVIS_BUILD_DIR/tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem &> /dev/null & fi diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 7adc217f..097f81dc 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -7,6 +7,6 @@ pip install --upgrade pip if [[ -n ${TOXENV} ]]; then pip install --upgrade tox else - pip install -e .[test] + pip install .[test] pip install --upgrade codecov fi diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh index 83d1731e..cc1061fe 100755 --- a/.ci/travis_script.sh +++ b/.ci/travis_script.sh @@ -4,8 +4,14 @@ set -e -x if [[ -n ${TOXENV} ]]; then tox -e ${TOXENV} -elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb ]]; then - pytest -v --database-backend=mongodb --cov=bigchaindb +elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ + -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then + # Run the full suite of tests for MongoDB over an unsecure connection + pytest -sv --database-backend=mongodb --cov=bigchaindb +elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ + "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then + # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'. + pytest -sv --database-backend=mongodb-ssl --cov=bigchaindb -m bdb_ssl else - pytest -v -n auto --cov=bigchaindb + pytest -sv -n auto --cov=bigchaindb fi diff --git a/.travis.yml b/.travis.yml index 9fc4e278..430f6aeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python cache: pip python: - - 3.4 - 3.5 - 3.6 @@ -14,12 +13,6 @@ env: matrix: fast_finish: true exclude: - - python: 3.4 - env: TOXENV=flake8 - - python: 3.4 - env: TOXENV=docsroot - - python: 3.4 - env: TOXENV=docsserver - python: 3.5 env: TOXENV=flake8 - python: 3.5 @@ -27,22 +20,30 @@ matrix: - python: 3.5 env: TOXENV=docsserver include: - - python: 3.4 - addons: - rethinkdb: '2.3.5' - env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb - python: 3.5 addons: rethinkdb: '2.3.5' env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb - python: 3.5 - env: BIGCHAINDB_DATABASE_BACKEND=mongodb + env: + - BIGCHAINDB_DATABASE_BACKEND=mongodb + - BIGCHAINDB_DATABASE_SSL= - python: 3.6 addons: rethinkdb: '2.3.5' env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb - python: 3.6 - env: BIGCHAINDB_DATABASE_BACKEND=mongodb + env: + - BIGCHAINDB_DATABASE_BACKEND=mongodb + - BIGCHAINDB_DATABASE_SSL= + - python: 3.5 + env: + - BIGCHAINDB_DATABASE_BACKEND=mongodb + - BIGCHAINDB_DATABASE_SSL=true + - python: 3.6 + env: + - BIGCHAINDB_DATABASE_BACKEND=mongodb + - BIGCHAINDB_DATABASE_SSL=true before_install: sudo .ci/travis-before-install.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bfbd8dd..c708e9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Change Log (Release Notes) All _notable_ changes to this project will be documented in this file (`CHANGELOG.md`). -This project adheres to [Semantic Versioning](http://semver.org/) (or at least we try). +This project adheres to [the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) (or at least we try). Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/). Note that each version (or "release") is the name of a [Git _tag_](https://git-scm.com/book/en/v2/Git-Basics-Tagging) of a particular commit, so the associated date and time are the date and time of that commit (as reported by GitHub), _not_ the "Uploaded on" date listed on PyPI (which may differ). For reference, the possible headings are: @@ -15,14 +15,77 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [1.0.0rc1] - 2017-06-23 +Tag name: v1.0.0rc1 + +### Added +* Support for secure TLS/SSL communication between MongoDB and {BigchainDB, MongoDB Backup Agent, MongoDB Monitoring Agent}. Pull Requests +[#1456](https://github.com/bigchaindb/bigchaindb/pull/1456), +[#1497](https://github.com/bigchaindb/bigchaindb/pull/1497), +[#1510](https://github.com/bigchaindb/bigchaindb/pull/1510), +[#1536](https://github.com/bigchaindb/bigchaindb/pull/1536), +[#1551](https://github.com/bigchaindb/bigchaindb/pull/1551) and +[#1552](https://github.com/bigchaindb/bigchaindb/pull/1552). +* Text search support (only if using MongoDB). Pull Requests [#1469](https://github.com/bigchaindb/bigchaindb/pull/1469) and [#1471](https://github.com/bigchaindb/bigchaindb/pull/1471) +* The `database.connection_timeout` configuration setting now works with RethinkDB too. [#1512](https://github.com/bigchaindb/bigchaindb/pull/1512) +* New code and tools for benchmarking CREATE transactions. [Pull Request #1511](https://github.com/bigchaindb/bigchaindb/pull/1511) + +### Changed +* There's an upgrade guide in `docs/upgrade-guides/v0.10-->v1.0.md`. It only covers changes to the transaction model and HTTP API. If that file hasn't been merged yet, see [Pull Request #1547](https://github.com/bigchaindb/bigchaindb/pull/1547) +* Cryptographic signatures now sign the whole (serialized) transaction body, including the transaction ID, but with all `"fulfillment"` values changed to `None`. [Pull Request #1225](https://github.com/bigchaindb/bigchaindb/pull/1225) +* In transactions, the value of `"amount"` must be a string. (Before, it was supposed to be a number.) [Pull Request #1286](https://github.com/bigchaindb/bigchaindb/pull/1286) +* In `setup.py`, the "Development Status" (as reported on PyPI) was changed from Alpha to Beta. [Pull Request #1437](https://github.com/bigchaindb/bigchaindb/pull/1437) +* If you explicitly specify a config file, e.g. `bigchaindb -c path/to/config start` and that file can't be found, then BigchainDB Server will fail with a helpful error message. [Pull Request #1486](https://github.com/bigchaindb/bigchaindb/pull/1486) +* Reduced the response time on the HTTP API endpoint to get all the unspent outputs associated with a given public key (a.k.a. "fast unspents"). [Pull Request #1411](https://github.com/bigchaindb/bigchaindb/pull/1411) +* Internally, the value of an asset's `"data"` is now stored in a separate assets table. This enabled the new text search. Each asset data is stored along with the associated CREATE transaction ID (asset ID). That data gets written when the containing block gets written to the bigchain table. [Pull Request #1460](https://github.com/bigchaindb/bigchaindb/pull/1460) +* Schema validation was sped up by switching to `rapidjson-schema`. [Pull Request #1494](https://github.com/bigchaindb/bigchaindb/pull/1494) +* If a node comes back from being down for a while, it will resume voting on blocks in the order determined by the MongoDB oplog, in the case of MongoDB. (In the case of RethinkDB, blocks missed in the changefeed will not be voted on.) [Pull Request #1389](https://github.com/bigchaindb/bigchaindb/pull/1389) +* Parallelized transaction schema validation in the vote pipeline. [Pull Request #1492](https://github.com/bigchaindb/bigchaindb/pull/1492) +* `asset.data` or `asset.id` are now *required* in a CREATE or TRANSFER transaction, respectively. [Pull Request #1518](https://github.com/bigchaindb/bigchaindb/pull/1518) +* The HTTP response body, in the response to the `GET /` and the `GET /api/v1` endpoints, was changed substantially. [Pull Request #1529](https://github.com/bigchaindb/bigchaindb/pull/1529) +* Changed the HTTP `GET /api/v1/transactions/{transaction_id}` endpoint. It now only returns the transaction if it's in a valid block. It also returns a new header with a relative link to a status monitor. [Pull Request #1543](https://github.com/bigchaindb/bigchaindb/pull/1543) +* All instances of `txid` and `tx_id` were replaced with `transaction_id`, in the transaction model and the HTTP API. [Pull Request #1532](https://github.com/bigchaindb/bigchaindb/pull/1532) +* The hostname and port were removed from all URLs in all HTTP API responses. [Pull Request #1538](https://github.com/bigchaindb/bigchaindb/pull/1538) +* Relative links were replaced with JSON objects in HTTP API responses. [Pull Request #1541](https://github.com/bigchaindb/bigchaindb/pull/1541) +* In the outputs endpoint of the HTTP API, the query parameter `unspent` was changed to `spent` (so no more double negatives). If that query parameter isn't included, then all outputs matching the specificed public key will be returned. If `spent=true`, then only the spent outputs will be returned. If `spent=false`, then only the unspent outputs will be returned. [Pull Request #1545](https://github.com/bigchaindb/bigchaindb/pull/1545) +* The supported crypto-conditions changed from version 01 of the crypto-conditions spec to version 02. [Pull Request #1562](https://github.com/bigchaindb/bigchaindb/pull/1562) +* The value of "version" inside a transaction must now be "1.0". (Before, it could be "0.anything".) [Pull Request #1574](https://github.com/bigchaindb/bigchaindb/pull/1574) + +### Removed +* The `server.threads` configuration setting (for the Gunicorn HTTP server) was removed from the default set of BigchainDB configuration settings. [Pull Request #1488](https://github.com/bigchaindb/bigchaindb/pull/1488) + +### Fixed +* The `GET /api/v1/outputs` endpoint was failing for some transactions with threshold conditions. Fixed in [Pull Request #1450](https://github.com/bigchaindb/bigchaindb/pull/1450) + +### External Contributors +* @elopio - Pull Requests [#1415](https://github.com/bigchaindb/bigchaindb/pull/1415) and [#1491](https://github.com/bigchaindb/bigchaindb/pull/1491) +* @CsterKuroi - [Pull Request #1447](https://github.com/bigchaindb/bigchaindb/pull/1447) +* @tdsgit - [Pull Request #1512](https://github.com/bigchaindb/bigchaindb/pull/1512) +* @lavinasachdev3 - [Pull Request #1357](https://github.com/bigchaindb/bigchaindb/pull/1357) + +### Notes +* We dropped support for Python 3.4. [Pull Request #1564](https://github.com/bigchaindb/bigchaindb/pull/1564) +* There were many improvements to our Kubernetes-based production deployment template (and the associated documentaiton). +* There is now a [BigchainDB Ruby driver](https://github.com/LicenseRocks/bigchaindb_ruby), created by @addywaddy at [license.rocks](https://github.com/bigchaindb/bigchaindb/pull/1437). +* The [BigchainDB JavaScript driver](https://github.com/bigchaindb/js-bigchaindb-driver) was moved to a different GitHub repo and is now officially maintained by the BigchainDB team. +* We continue to recommend using MongoDB. + +## [0.10.3] - 2017-06-29 +Tag name: v0.10.3 + +## Fixed +* Pin minor+ version of `cryptoconditions` to avoid upgrading to a non + compatible version. +[commit 97268a5](https://github.com/bigchaindb/bigchaindb/commit/97268a577bf27942a87d8eb838f4816165c84fd5) + ## [0.10.2] - 2017-05-16 Tag name: v0.10.2 -## Added +### Added * Add Cross Origin Resource Sharing (CORS) support for the HTTP API. [Commit 6cb7596](https://github.com/bigchaindb/bigchaindb/commit/6cb75960b05403c77bdae0fd327612482589efcb) -## Fixed +### Fixed * Fixed `streams_v1` API link in response to `GET /api/v1`. [Pull Request #1466](https://github.com/bigchaindb/bigchaindb/pull/1466) * Fixed mismatch between docs and implementation for `GET /blocks?status=` @@ -32,10 +95,10 @@ Tag name: v0.10.2 ## [0.10.1] - 2017-04-19 Tag name: v0.10.1 -## Added +### Added * Documentation for the BigchainDB settings `wsserver.host` and `wsserver.port`. [Pull Request #1408](https://github.com/bigchaindb/bigchaindb/pull/1408) -## Fixed +### Fixed * Fixed `Dockerfile`, which was failing to build. It now starts `FROM python:3.6` (instead of `FROM ubuntu:xenial`). [Pull Request #1410](https://github.com/bigchaindb/bigchaindb/pull/1410) * Fixed the `Makefile` so that `release` depends on `dist`. [Pull Request #1405](https://github.com/bigchaindb/bigchaindb/pull/1405) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 840a0895..1b8fc9c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ Familiarize yourself with how we do coding and documentation in the BigchainDB p ### Step 2 - Install some Dependencies * [Install RethinkDB Server](https://rethinkdb.com/docs/install/) -* Make sure you have Python 3.4+ (preferably in a virtualenv) +* Make sure you have Python 3.5+ (preferably in a virtualenv) * [Install BigchaindB Server's OS-level dependencies](https://docs.bigchaindb.com/projects/server/en/latest/appendices/install-os-level-deps.html) * [Make sure you have the latest Python 3 version of pip and setuptools](https://docs.bigchaindb.com/projects/server/en/latest/appendices/install-latest-pip.html) diff --git a/Dockerfile b/Dockerfile index 807761fe..bd3c8d9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,10 +8,11 @@ RUN apt-get -qq update \ && pip install --no-cache-dir . \ && apt-get autoremove \ && apt-get clean -VOLUME ["/data"] +VOLUME ["/data", "/certs"] WORKDIR /data ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_SCHEME ws ENTRYPOINT ["bigchaindb"] CMD ["start"] diff --git a/Dockerfile-dev b/Dockerfile-dev index 17c8b073..8241346f 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -3,19 +3,19 @@ LABEL maintainer "dev@bigchaindb.com" RUN apt-get update \ && apt-get install -y vim \ + && pip install -U pip \ && pip install pynacl \ && apt-get autoremove \ && apt-get clean -VOLUME ["/data"] -WORKDIR /data - -ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_SCHEME ws + +ARG backend RUN mkdir -p /usr/src/app COPY . /usr/src/app/ WORKDIR /usr/src/app RUN pip install --no-cache-dir -e .[dev] -RUN bigchaindb -y configure mongodb +RUN bigchaindb -y configure "$backend" diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index 5ca44e83..3fa35391 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -6,7 +6,7 @@ This guide starts out with our general Python coding style guidelines and ends w Our starting point is [PEP8](https://www.python.org/dev/peps/pep-0008/), the standard "Style Guide for Python Code." Many Python IDEs will check your code against PEP8. (Note that PEP8 isn't frozen; it actually changes over time, but slowly.) -BigchainDB uses Python 3.4+, so you can ignore all PEP8 guidelines specific to Python 2. +BigchainDB uses Python 3.5+, so you can ignore all PEP8 guidelines specific to Python 2. ### Python Docstrings diff --git a/README.md b/README.md index e6d0a6a5..118d6f85 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ BigchainDB is a scalable blockchain database. [The whitepaper](https://www.bigch * [Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) * [Blog](https://medium.com/the-bigchaindb-blog) * [Twitter](https://twitter.com/BigchainDB) -* [Google Group](https://groups.google.com/forum/#!forum/bigchaindb) ## Links for Developers diff --git a/Release_Process.md b/RELEASE_PROCESS.md similarity index 90% rename from Release_Process.md rename to RELEASE_PROCESS.md index be4c448a..a12756a0 100644 --- a/Release_Process.md +++ b/RELEASE_PROCESS.md @@ -2,8 +2,14 @@ The release process for BigchainDB server differs slightly depending on whether it's a minor or a patch release. -BigchainDB follows [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH), taking into account -that [major version 0.x does not export a stable API](http://semver.org/#spec-item-4). +BigchainDB follows +[the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) +(i.e. MAJOR.MINOR.PATCH), +which is almost identical +to [regular semantic versioning](http://semver.org/) +except release candidates are labelled like +`3.4.5rc2` not `3.4.5-rc2` (with no hyphen). + ## Minor release @@ -14,6 +20,7 @@ A minor release is preceeded by a feature freeze and created from the 'master' b 1. Create and checkout a new branch for the minor release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` (*not* 0.9.0, this new branch will be for e.g. 0.9.0, 0.9.1, 0.9.2, etc. each of which will be identified by a tagged commit) 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) 1. Commit that change, and push the new branch to GitHub +1. On GitHub, use the new branch to create a new pull request and wait for all the tests to pass 1. Follow steps outlined in [Common Steps](#common-steps) 1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev`. This is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available). 1. Go to [Docker Hub](https://hub.docker.com/), sign in, go to Settings - Build Settings, and under the build with Docker Tag Name equal to `latest`, change the Name to the number of the new release, e.g. `0.9` diff --git a/benchmark.yml b/benchmark.yml new file mode 100644 index 00000000..c7319040 --- /dev/null +++ b/benchmark.yml @@ -0,0 +1,37 @@ +version: '2' + +services: + bdb: + build: + context: . + dockerfile: Dockerfile-dev + args: + backend: mongodb + volumes: + - ./bigchaindb:/usr/src/app/bigchaindb + - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./k8s:/usr/src/app/k8s + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini + - ./scripts:/usr/src/app/scripts + environment: + BIGCHAINDB_DATABASE_BACKEND: mongodb + BIGCHAINDB_DATABASE_HOST: mdb + BIGCHAINDB_DATABASE_PORT: 27017 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + BIGCHAINDB_GRAPHITE_HOST: graphite + ports: + - "9984" + command: bigchaindb start + + graphite: + image: hopsoft/graphite-statsd + ports: + - "2003-2004" + - "2023-2024" + - "8125/udp" + - "8126" + - "80" diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 07f35320..1bef9c17 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -30,7 +30,6 @@ _base_database_mongodb = { 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), 'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'), 'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD') } @@ -46,6 +45,12 @@ _database_mongodb = { 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), 'connection_timeout': 5000, 'max_tries': 3, + 'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)), + 'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT'), + 'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE'), + 'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE'), + 'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE'), + 'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE') } _database_mongodb.update(_base_database_mongodb) @@ -64,6 +69,7 @@ config = { 'workers': None, # if none, the value will be cpu_count * 2 + 1 }, 'wsserver': { + 'scheme': os.environ.get('BIGCHAINDB_WSSERVER_SCHEME') or 'ws', 'host': os.environ.get('BIGCHAINDB_WSSERVER_HOST') or 'localhost', 'port': int(os.environ.get('BIGCHAINDB_WSSERVER_PORT', 9985)), }, @@ -89,6 +95,9 @@ config = { 'fmt_logfile': log_config['formatters']['file']['format'], 'granular_levels': {}, }, + 'graphite': { + 'host': os.environ.get('BIGCHAINDB_GRAPHITE_HOST', 'localhost'), + }, } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index b717703b..61fc52d8 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -16,7 +16,9 @@ logger = logging.getLogger(__name__) def connect(backend=None, host=None, port=None, name=None, max_tries=None, - connection_timeout=None, replicaset=None, ssl=None, login=None, password=None): + connection_timeout=None, replicaset=None, ssl=None, login=None, password=None, + ca_cert=None, certfile=None, keyfile=None, keyfile_passphrase=None, + crlfile=None): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -38,6 +40,8 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, :exc:`~ConnectionError`: If the connection to the database fails. :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend` is not supported or could not be loaded. + :exc:`~AuthenticationError`: If there is a OperationFailure due to + Authentication failure after connecting to the database. """ backend = backend or bigchaindb.config['database']['backend'] @@ -53,6 +57,11 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False) login = login or bigchaindb.config['database'].get('login') password = password or bigchaindb.config['database'].get('password') + ca_cert = ca_cert or bigchaindb.config['database'].get('ca_cert', None) + certfile = certfile or bigchaindb.config['database'].get('certfile', None) + keyfile = keyfile or bigchaindb.config['database'].get('keyfile', None) + keyfile_passphrase = keyfile_passphrase or bigchaindb.config['database'].get('keyfile_passphrase', None) + crlfile = crlfile or bigchaindb.config['database'].get('crlfile', None) try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -66,13 +75,15 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None, logger.debug('Connection: {}'.format(Class)) return Class(host=host, port=port, dbname=dbname, max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset, ssl=ssl, login=login, password=password) + replicaset=replicaset, ssl=ssl, login=login, password=password, + ca_cert=ca_cert, certfile=certfile, keyfile=keyfile, + keyfile_passphrase=keyfile_passphrase, crlfile=crlfile) class Connection: """Connection class interface. - All backend implementations should provide a connection class that + All backend implementations should provide a connection class that inherits from and implements this class. """ diff --git a/bigchaindb/backend/exceptions.py b/bigchaindb/backend/exceptions.py index e59b317b..017e19e4 100644 --- a/bigchaindb/backend/exceptions.py +++ b/bigchaindb/backend/exceptions.py @@ -9,10 +9,6 @@ class ConnectionError(BackendError): """Exception raised when the connection to the backend fails.""" -class AuthenticationError(ConnectionError): - """Exception raised when MongoDB Authentication fails""" - - class OperationError(BackendError): """Exception raised when a backend operation fails.""" diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 8ed5077e..338ee18e 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -1,5 +1,6 @@ import time import logging +from ssl import CERT_REQUIRED import pymongo @@ -8,8 +9,7 @@ from bigchaindb.utils import Lazy from bigchaindb.common.exceptions import ConfigurationError from bigchaindb.backend.exceptions import (DuplicateKeyError, OperationError, - ConnectionError, - AuthenticationError) + ConnectionError) from bigchaindb.backend.connection import Connection logger = logging.getLogger(__name__) @@ -17,7 +17,10 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): - def __init__(self, replicaset=None, ssl=None, login=None, password=None, **kwargs): + def __init__(self, replicaset=None, ssl=None, login=None, password=None, + ca_cert=None, certfile=None, keyfile=None, + keyfile_passphrase=None, crlfile=None, **kwargs): + """Create a new Connection instance. Args: @@ -32,6 +35,11 @@ class MongoDBConnection(Connection): self.ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False) self.login = login or bigchaindb.config['database'].get('login') self.password = password or bigchaindb.config['database'].get('password') + self.ca_cert = ca_cert or bigchaindb.config['database'].get('ca_cert', None) + self.certfile = certfile or bigchaindb.config['database'].get('certfile', None) + self.keyfile = keyfile or bigchaindb.config['database'].get('keyfile', None) + self.keyfile_passphrase = keyfile_passphrase or bigchaindb.config['database'].get('keyfile_passphrase', None) + self.crlfile = crlfile or bigchaindb.config['database'].get('crlfile', None) @property def db(self): @@ -69,50 +77,114 @@ class MongoDBConnection(Connection): Raises: :exc:`~ConnectionError`: If the connection to the database fails. + :exc:`~AuthenticationError`: If there is a OperationFailure due to + Authentication failure after connecting to the database. + :exc:`~ConfigurationError`: If there is a ConfigurationError while + connecting to the database. """ try: # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout, - self.dbname, self.ssl, self.login, self.password) + initialize_replica_set(self.host, + self.port, + self.connection_timeout, + self.dbname, + self.ssl, + self.login, + self.password, + self.ca_cert, + self.certfile, + self.keyfile, + self.keyfile_passphrase, + self.crlfile) - # FYI: this might raise a `ServerSelectionTimeoutError`, - # that is a subclass of `ConnectionFailure`. - client = pymongo.MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout, - ssl=self.ssl) - - if self.login is not None and self.password is not None: - client[self.dbname].authenticate(self.login, self.password) + # FYI: the connection process might raise a + # `ServerSelectionTimeoutError`, that is a subclass of + # `ConnectionFailure`. + # The presence of ca_cert, certfile, keyfile, crlfile implies the + # use of certificates for TLS connectivity. + if self.ca_cert is None or self.certfile is None or \ + self.keyfile is None or self.crlfile is None: + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl) + if self.login is not None and self.password is not None: + client[self.dbname].authenticate(self.login, self.password) + else: + logger.info('Connecting to MongoDB over TLS/SSL...') + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl, + ssl_ca_certs=self.ca_cert, + ssl_certfile=self.certfile, + ssl_keyfile=self.keyfile, + ssl_pem_passphrase=self.keyfile_passphrase, + ssl_crlfile=self.crlfile, + ssl_cert_reqs=CERT_REQUIRED) + if self.login is not None: + client[self.dbname].authenticate(self.login, + mechanism='MONGODB-X509') return client - # `initialize_replica_set` might raise `ConnectionFailure` or `OperationFailure`. + # `initialize_replica_set` might raise `ConnectionFailure`, + # `OperationFailure` or `ConfigurationError`. except (pymongo.errors.ConnectionFailure, pymongo.errors.OperationFailure) as exc: - if "Authentication fail" in str(exc): - raise AuthenticationError() from exc - raise ConnectionError() from exc + logger.info('Exception in _connect(): {}'.format(exc)) + raise ConnectionError(str(exc)) from exc + except pymongo.errors.ConfigurationError as exc: + raise ConfigurationError from exc -def initialize_replica_set(host, port, connection_timeout, dbname, ssl, login, password): +def initialize_replica_set(host, port, connection_timeout, dbname, ssl, login, + password, ca_cert, certfile, keyfile, + keyfile_passphrase, crlfile): """Initialize a replica set. If already initialized skip.""" # Setup a MongoDB connection # The reason we do this instead of `backend.connect` is that # `backend.connect` will connect you to a replica set but this fails if # you try to connect to a replica set that is not yet initialized - conn = pymongo.MongoClient(host=host, - port=port, - serverselectiontimeoutms=connection_timeout, - ssl=ssl) + try: + # The presence of ca_cert, certfile, keyfile, crlfile implies the + # use of certificates for TLS connectivity. + if ca_cert is None or certfile is None or keyfile is None or \ + crlfile is None: + conn = pymongo.MongoClient(host, + port, + serverselectiontimeoutms=connection_timeout, + ssl=ssl) + if login is not None and password is not None: + conn[dbname].authenticate(login, password) + else: + logger.info('Connecting to MongoDB over TLS/SSL...') + conn = pymongo.MongoClient(host, + port, + serverselectiontimeoutms=connection_timeout, + ssl=ssl, + ssl_ca_certs=ca_cert, + ssl_certfile=certfile, + ssl_keyfile=keyfile, + ssl_pem_passphrase=keyfile_passphrase, + ssl_crlfile=crlfile, + ssl_cert_reqs=CERT_REQUIRED) + if login is not None: + logger.info('Authenticating to the database...') + conn[dbname].authenticate(login, mechanism='MONGODB-X509') - if login is not None and password is not None: - conn[dbname].authenticate(login, password) + except (pymongo.errors.ConnectionFailure, + pymongo.errors.OperationFailure) as exc: + logger.info('Exception in _connect(): {}'.format(exc)) + raise ConnectionError(str(exc)) from exc + except pymongo.errors.ConfigurationError as exc: + raise ConfigurationError from exc _check_replica_set(conn) host = '{}:{}'.format(bigchaindb.config['database']['host'], @@ -129,6 +201,10 @@ def initialize_replica_set(host, port, connection_timeout, dbname, ssl, login, p else: _wait_for_replica_set_initialization(conn) logger.info('Initialized replica set') + finally: + if conn is not None: + logger.info('Closing initial connection to MongoDB') + conn.close() def _check_replica_set(conn): diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 3bc4dc53..673f643f 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -157,8 +157,8 @@ def get_spent(conn, transaction_id, output): {'$match': { 'block.transactions.inputs': { '$elemMatch': { - 'fulfills.txid': transaction_id, - 'fulfills.output': output, + 'fulfills.transaction_id': transaction_id, + 'fulfills.output_index': output, }, }, }}, @@ -166,8 +166,8 @@ def get_spent(conn, transaction_id, output): {'$match': { 'block.transactions.inputs': { '$elemMatch': { - 'fulfills.txid': transaction_id, - 'fulfills.output': output, + 'fulfills.transaction_id': transaction_id, + 'fulfills.output_index': output, }, }, }}, diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 6c54bfd8..e398560f 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -68,11 +68,11 @@ def create_bigchain_secondary_index(conn, dbname): .create_index('block.transactions.outputs.public_keys', name='outputs') - # secondary index on inputs/transaction links (txid, output) + # secondary index on inputs/transaction links (transaction_id, output) conn.conn[dbname]['bigchain']\ .create_index([ - ('block.transactions.inputs.fulfills.txid', ASCENDING), - ('block.transactions.inputs.fulfills.output', ASCENDING), + ('block.transactions.inputs.fulfills.transaction_id', ASCENDING), + ('block.transactions.inputs.fulfills.output_index', ASCENDING), ], name='inputs') diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index 3ee90f34..cac9cc94 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -122,7 +122,8 @@ def get_spent(connection, transaction_id, output): .get_all([transaction_id, output], index='inputs') .concat_map(lambda doc: doc['block']['transactions']) .filter(lambda transaction: transaction['inputs'].contains( - lambda input_: input_['fulfills'] == {'txid': transaction_id, 'output': output}))) + lambda input_: input_['fulfills'] == { + 'transaction_id': transaction_id, 'output_index': output}))) @register_query(RethinkDBConnection) @@ -286,7 +287,8 @@ def unwind_block_transactions(block): def get_spending_transactions(connection, links): query = ( r.table('bigchain') - .get_all(*[(l['txid'], l['output']) for l in links], index='inputs') + .get_all(*[(l['transaction_id'], l['output_index']) for l in links], + index='inputs') .concat_map(unwind_block_transactions) # filter transactions spending output .filter(lambda doc: r.expr(links).set_intersection( diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index 8f0f6b9c..ea6f4e25 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -79,16 +79,16 @@ def create_bigchain_secondary_index(connection, dbname): .concat_map(lambda tx: tx['outputs']['public_keys']) .reduce(lambda l, r: l + r), multi=True)) - # secondary index on inputs/transaction links (txid, output) + # secondary index on inputs/transaction links (transaction_id, output) connection.run( r.db(dbname) .table('bigchain') .index_create('inputs', r.row['block']['transactions'] .concat_map(lambda tx: tx['inputs']['fulfills']) - .with_fields('txid', 'output') - .map(lambda fulfills: [fulfills['txid'], - fulfills['output']]), + .with_fields('transaction_id', 'output_index') + .map(lambda fulfills: [fulfills['transaction_id'], + fulfills['output_index']]), multi=True)) # wait for rethinkdb to finish creating secondary indexes diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index a46019da..146dab91 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -96,7 +96,7 @@ def run_configure(args, skip_if_exists=False): val = conf['server'][key] conf['server'][key] = input_on_stderr('API Server {}? (default `{}`): '.format(key, val), val) - for key in ('host', 'port'): + for key in ('scheme', 'host', 'port'): val = conf['wsserver'][key] conf['wsserver'][key] = input_on_stderr('WebSocket Server {}? (default `{}`): '.format(key, val), val) diff --git a/bigchaindb/common/exceptions.py b/bigchaindb/common/exceptions.py index 258001b8..ec4c9702 100644 --- a/bigchaindb/common/exceptions.py +++ b/bigchaindb/common/exceptions.py @@ -106,3 +106,7 @@ class SybilError(ValidationError): class DuplicateTransaction(ValidationError): """Raised if a duplicated transaction is found""" + + +class ThresholdTooDeep(ValidationError): + """Raised if threshold condition is too deep""" diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index f63b652e..72b51cf1 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -56,7 +56,7 @@ properties: See: `Metadata`_. version: type: string - pattern: "^0\\." + pattern: "^1\\.0$" description: | BigchainDB transaction schema version. definitions: @@ -150,11 +150,10 @@ definitions: - uri properties: details: - type: object - additionalProperties: true + "$ref": "#/definitions/condition_details" uri: type: string - pattern: "^cc:([1-9a-f][0-9a-f]{0,3}|0):[1-9a-f][0-9a-f]{0,15}:[a-zA-Z0-9_-]{0,86}:([1-9][0-9]{0,17}|0)$" + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})?(.+)$" public_keys: "$ref": "#/definitions/public_keys" description: | @@ -174,28 +173,14 @@ definitions: description: | List of public keys of the previous owners of the asset. fulfillment: + description: | + Fulfillment of an `Output.condition`_, or, put a different way, a payload + that satisfies the condition of a previous output to prove that the + creator(s) of this transaction have control over the listed asset. anyOf: - - type: object - additionalProperties: false - properties: - bitmask: - type: integer - public_key: - type: string - type: - type: string - signature: - anyOf: - - type: string - - type: 'null' - type_id: - type: integer - description: | - Fulfillment of an `Output.condition`_, or, put a different way, a payload - that satisfies the condition of a previous output to prove that the - creator(s) of this transaction have control over the listed asset. - type: string - pattern: "^cf:([1-9a-f][0-9a-f]{0,3}|0):[a-zA-Z0-9_-]*$" + pattern: "^[a-zA-Z0-9_-]*$" + - "$ref": "#/definitions/condition_details" fulfills: anyOf: - type: 'object' @@ -203,14 +188,14 @@ definitions: Reference to the output that is being spent. additionalProperties: false required: - - output - - txid + - output_index + - transaction_id properties: - output: + output_index: "$ref": "#/definitions/offset" description: | Index of the output containing the condition being fulfilled - txid: + transaction_id: "$ref": "#/definitions/sha3_hexdigest" description: | Transaction ID containing the output to spend @@ -224,3 +209,37 @@ definitions: additionalProperties: true minProperties: 1 - type: 'null' + condition_details: + description: | + Details needed to reconstruct the condition associated with an output. + Currently, BigchainDB only supports ed25519 and threshold condition types. + anyOf: + - type: object + additionalProperties: false + required: + - type + - public_key + properties: + type: + type: string + pattern: "^ed25519-sha-256$" + public_key: + "$ref": "#/definitions/base58" + - type: object + additionalProperties: false + required: + - type + - threshold + - subconditions + properties: + type: + type: "string" + pattern: "^threshold-sha-256$" + threshold: + type: integer + minimum: 1 + maximum: 100 + subconditions: + type: array + items: + "$ref": "#/definitions/condition_details" diff --git a/bigchaindb/common/schema/transaction_create.yaml b/bigchaindb/common/schema/transaction_create.yaml index 2383a102..3d393347 100644 --- a/bigchaindb/common/schema/transaction_create.yaml +++ b/bigchaindb/common/schema/transaction_create.yaml @@ -14,6 +14,8 @@ properties: - type: object additionalProperties: true - type: 'null' + required: + - data inputs: type: array title: "Transaction inputs" diff --git a/bigchaindb/common/schema/transaction_transfer.yaml b/bigchaindb/common/schema/transaction_transfer.yaml index 09a5aa1b..b8b79696 100644 --- a/bigchaindb/common/schema/transaction_transfer.yaml +++ b/bigchaindb/common/schema/transaction_transfer.yaml @@ -12,6 +12,8 @@ properties: "$ref": "#/definitions/sha3_hexdigest" description: | ID of the transaction that created the asset. + required: + - id inputs: type: array title: "Transaction inputs" diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 365fee8f..a377b994 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1,16 +1,17 @@ from copy import deepcopy from functools import reduce -from cryptoconditions import (Fulfillment, ThresholdSha256Fulfillment, - Ed25519Fulfillment) -from cryptoconditions.exceptions import ParsingError +import base58 +from cryptoconditions import Fulfillment, ThresholdSha256, Ed25519Sha256 +from cryptoconditions.exceptions import ( + ParsingError, ASN1DecodeError, ASN1EncodeError, UnsupportedTypeError) from bigchaindb.common.crypto import PrivateKey, hash_data from bigchaindb.common.exceptions import (KeypairMismatchException, InvalidHash, InvalidSignature, - AmountError, AssetIdMismatch) -from bigchaindb.common.utils import serialize, gen_timestamp -import bigchaindb.version + AmountError, AssetIdMismatch, + ThresholdTooDeep) +from bigchaindb.common.utils import serialize class Input(object): @@ -65,16 +66,8 @@ class Input(object): """ try: fulfillment = self.fulfillment.serialize_uri() - except (TypeError, AttributeError): - # NOTE: When a non-signed transaction is casted to a dict, - # `self.inputs` value is lost, as in the node's - # transaction model that is saved to the database, does not - # account for its dictionary form but just for its signed uri - # form. - # Hence, when a non-signed fulfillment is to be cast to a - # dict, we just call its internal `to_dict` method here and - # its `from_dict` method in `Fulfillment.from_dict`. - fulfillment = self.fulfillment.to_dict() + except (TypeError, AttributeError, ASN1EncodeError): + fulfillment = _fulfillment_to_details(self.fulfillment) try: # NOTE: `self.fulfills` can be `None` and that's fine @@ -114,19 +107,74 @@ class Input(object): Raises: InvalidSignature: If an Input's URI couldn't be parsed. """ - try: - fulfillment = Fulfillment.from_uri(data['fulfillment']) - except ValueError: - # TODO FOR CC: Throw an `InvalidSignature` error in this case. - raise InvalidSignature("Fulfillment URI couldn't been parsed") - except TypeError: - # NOTE: See comment about this special case in - # `Input.to_dict` - fulfillment = Fulfillment.from_dict(data['fulfillment']) + fulfillment = data['fulfillment'] + if not isinstance(fulfillment, Fulfillment): + try: + fulfillment = Fulfillment.from_uri(data['fulfillment']) + except ASN1DecodeError: + # TODO Remove as it is legacy code, and simply fall back on + # ASN1DecodeError + raise InvalidSignature("Fulfillment URI couldn't been parsed") + except TypeError: + # NOTE: See comment about this special case in + # `Input.to_dict` + fulfillment = _fulfillment_from_details(data['fulfillment']) fulfills = TransactionLink.from_dict(data['fulfills']) return cls(fulfillment, data['owners_before'], fulfills) +def _fulfillment_to_details(fulfillment): + """ + Encode a fulfillment as a details dictionary + + Args: + fulfillment: Crypto-conditions Fulfillment object + """ + + if fulfillment.type_name == 'ed25519-sha-256': + return { + 'type': 'ed25519-sha-256', + 'public_key': base58.b58encode(fulfillment.public_key), + } + + if fulfillment.type_name == 'threshold-sha-256': + subconditions = [ + _fulfillment_to_details(cond['body']) + for cond in fulfillment.subconditions + ] + return { + 'type': 'threshold-sha-256', + 'threshold': fulfillment.threshold, + 'subconditions': subconditions, + } + + raise UnsupportedTypeError(fulfillment.type_name) + + +def _fulfillment_from_details(data): + """ + Load a fulfillment for a signing spec dictionary + + Args: + data: tx.output[].condition.details dictionary + """ + if data['type'] == 'ed25519-sha-256': + public_key = base58.b58decode(data['public_key']) + return Ed25519Sha256(public_key=public_key) + + if data['type'] == 'threshold-sha-256': + try: + threshold = ThresholdSha256(data['threshold']) + for cond in data['subconditions']: + cond = _fulfillment_from_details(cond) + threshold.add_subfulfillment(cond) + return threshold + except RecursionError: + raise ThresholdTooDeep() + + raise UnsupportedTypeError(data.get('type')) + + class TransactionLink(object): """An object for unidirectional linking to a Transaction's Output. @@ -175,7 +223,7 @@ class TransactionLink(object): :class:`~bigchaindb.common.transaction.TransactionLink` """ try: - return cls(link['txid'], link['output']) + return cls(link['transaction_id'], link['output_index']) except TypeError: return cls() @@ -189,8 +237,8 @@ class TransactionLink(object): return None else: return { - 'txid': self.txid, - 'output': self.output, + 'transaction_id': self.txid, + 'output_index': self.output, } def to_uri(self, path=''): @@ -259,7 +307,7 @@ class Output(object): # and fulfillment! condition = {} try: - condition['details'] = self.fulfillment.to_dict() + condition['details'] = _fulfillment_to_details(self.fulfillment) except AttributeError: pass @@ -310,13 +358,14 @@ class Output(object): raise ValueError('`public_keys` needs to contain at least one' 'owner') elif len(public_keys) == 1 and not isinstance(public_keys[0], list): - try: - ffill = Ed25519Fulfillment(public_key=public_keys[0]) - except TypeError: + if isinstance(public_keys[0], Fulfillment): ffill = public_keys[0] + else: + ffill = Ed25519Sha256( + public_key=base58.b58decode(public_keys[0])) return cls(ffill, public_keys, amount=amount) else: - initial_cond = ThresholdSha256Fulfillment(threshold=threshold) + initial_cond = ThresholdSha256(threshold=threshold) threshold_cond = reduce(cls._gen_condition, public_keys, initial_cond) return cls(threshold_cond, public_keys, amount=amount) @@ -331,13 +380,13 @@ class Output(object): :meth:`~.Output.generate`. Args: - initial (:class:`cryptoconditions.ThresholdSha256Fulfillment`): + initial (:class:`cryptoconditions.ThresholdSha256`): A Condition representing the overall root. new_public_keys (:obj:`list` of :obj:`str`|str): A list of new owners or a single new owner. Returns: - :class:`cryptoconditions.ThresholdSha256Fulfillment`: + :class:`cryptoconditions.ThresholdSha256`: """ try: threshold = len(new_public_keys) @@ -345,7 +394,7 @@ class Output(object): threshold = None if isinstance(new_public_keys, list) and len(new_public_keys) > 1: - ffill = ThresholdSha256Fulfillment(threshold=threshold) + ffill = ThresholdSha256(threshold=threshold) reduce(cls._gen_condition, new_public_keys, ffill) elif isinstance(new_public_keys, list) and len(new_public_keys) <= 1: raise ValueError('Sublist cannot contain single owner') @@ -354,16 +403,17 @@ class Output(object): new_public_keys = new_public_keys.pop() except AttributeError: pass - try: - ffill = Ed25519Fulfillment(public_key=new_public_keys) - except TypeError: - # NOTE: Instead of submitting base58 encoded addresses, a user - # of this class can also submit fully instantiated - # Cryptoconditions. In the case of casting - # `new_public_keys` to a Ed25519Fulfillment with the - # result of a `TypeError`, we're assuming that - # `new_public_keys` is a Cryptocondition then. + # NOTE: Instead of submitting base58 encoded addresses, a user + # of this class can also submit fully instantiated + # Cryptoconditions. In the case of casting + # `new_public_keys` to a Ed25519Fulfillment with the + # result of a `TypeError`, we're assuming that + # `new_public_keys` is a Cryptocondition then. + if isinstance(new_public_keys, Fulfillment): ffill = new_public_keys + else: + ffill = Ed25519Sha256( + public_key=base58.b58decode(new_public_keys)) initial.add_subfulfillment(ffill) return initial @@ -384,7 +434,7 @@ class Output(object): :class:`~bigchaindb.common.transaction.Output` """ try: - fulfillment = Fulfillment.from_dict(data['condition']['details']) + fulfillment = _fulfillment_from_details(data['condition']['details']) except KeyError: # NOTE: Hashlock condition case fulfillment = data['condition']['uri'] @@ -415,13 +465,13 @@ class Transaction(object): ``id`` property. metadata (dict): Metadata to be stored along with the Transaction. - version (int): Defines the version number of a Transaction. + version (string): Defines the version number of a Transaction. """ CREATE = 'CREATE' TRANSFER = 'TRANSFER' GENESIS = 'GENESIS' ALLOWED_OPERATIONS = (CREATE, TRANSFER, GENESIS) - VERSION = '.'.join(bigchaindb.version.__short_version__.split('.')[:2]) + VERSION = '1.0' def __init__(self, operation, asset, inputs=None, outputs=None, metadata=None, version=None): @@ -441,7 +491,7 @@ class Transaction(object): lock. metadata (dict): Metadata to be stored along with the Transaction. - version (int): Defines the version number of a Transaction. + version (string): Defines the version number of a Transaction. """ if operation not in Transaction.ALLOWED_OPERATIONS: allowed_ops = ', '.join(self.__class__.ALLOWED_OPERATIONS) @@ -661,7 +711,7 @@ class Transaction(object): This method works only for the following Cryptoconditions currently: - Ed25519Fulfillment - - ThresholdSha256Fulfillment + - ThresholdSha256 Furthermore, note that all keys required to fully sign the Transaction have to be passed to this method. A subset of all will cause this method to fail. @@ -712,7 +762,7 @@ class Transaction(object): This method works only for the following Cryptoconditions currently: - Ed25519Fulfillment - - ThresholdSha256Fulfillment. + - ThresholdSha256. Args: input_ (:class:`~bigchaindb.common.transaction. @@ -720,10 +770,10 @@ class Transaction(object): message (str): The message to be signed key_pairs (dict): The keys to sign the Transaction with. """ - if isinstance(input_.fulfillment, Ed25519Fulfillment): + if isinstance(input_.fulfillment, Ed25519Sha256): return cls._sign_simple_signature_fulfillment(input_, message, key_pairs) - elif isinstance(input_.fulfillment, ThresholdSha256Fulfillment): + elif isinstance(input_.fulfillment, ThresholdSha256): return cls._sign_threshold_signature_fulfillment(input_, message, key_pairs) else: @@ -749,7 +799,10 @@ class Transaction(object): try: # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings - input_.fulfillment.sign(message.encode(), key_pairs[public_key]) + input_.fulfillment.sign( + message.encode(), + base58.b58decode(key_pairs[public_key].encode()), + ) except KeyError: raise KeypairMismatchException('Public key {} is not a pair to ' 'any of the private keys' @@ -758,7 +811,7 @@ class Transaction(object): @classmethod def _sign_threshold_signature_fulfillment(cls, input_, message, key_pairs): - """Signs a ThresholdSha256Fulfillment. + """Signs a ThresholdSha256. Args: input_ (:class:`~bigchaindb.common.transaction. @@ -778,7 +831,8 @@ class Transaction(object): # TODO FOR CC: `get_subcondition` is singular. One would not # expect to get a list back. ccffill = input_.fulfillment - subffills = ccffill.get_subcondition_from_vk(owner_before) + subffills = ccffill.get_subcondition_from_vk( + base58.b58decode(owner_before)) if not subffills: raise KeypairMismatchException('Public key {} cannot be found ' 'in the fulfillment' @@ -793,7 +847,7 @@ class Transaction(object): # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings for subffill in subffills: - subffill.sign(message.encode(), private_key) + subffill.sign(message.encode(), base58.b58decode(private_key.encode())) return input_ def inputs_valid(self, outputs=None): @@ -882,7 +936,8 @@ class Transaction(object): ccffill = input_.fulfillment try: parsed_ffill = Fulfillment.from_uri(ccffill.serialize_uri()) - except (TypeError, ValueError, ParsingError): + except (TypeError, ValueError, + ParsingError, ASN1DecodeError, ASN1EncodeError): return False if operation in (Transaction.CREATE, Transaction.GENESIS): @@ -897,8 +952,7 @@ class Transaction(object): # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings - ffill_valid = parsed_ffill.validate(message=tx_serialized.encode(), - now=gen_timestamp()) + ffill_valid = parsed_ffill.validate(message=tx_serialized.encode()) return output_valid and ffill_valid def to_dict(self): @@ -940,7 +994,7 @@ class Transaction(object): tx_dict = deepcopy(tx_dict) for input_ in tx_dict['inputs']: # NOTE: Not all Cryptoconditions return a `signature` key (e.g. - # ThresholdSha256Fulfillment), so setting it to `None` in any + # ThresholdSha256), so setting it to `None` in any # case could yield incorrect signatures. This is why we only # set it to `None` if it's set in the dict. input_['fulfillment'] = None diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 2f4d24da..9824e9fc 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,4 +1,5 @@ import random +import statsd from time import time from bigchaindb import exceptions as core_exceptions @@ -71,6 +72,8 @@ class Bigchain(object): if not self.me or not self.me_private: raise exceptions.KeypairNotFoundException() + self.statsd = statsd.StatsClient(bigchaindb.config['graphite']['host']) + federation = property(lambda self: set(self.nodes_except_me + [self.me])) """ Set of federation member public keys """ @@ -399,20 +402,33 @@ class Bigchain(object): :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s pointing to another transaction's condition """ - return self.get_outputs_filtered(owner, include_spent=False) + return self.get_outputs_filtered(owner, spent=False) @property def fastquery(self): return fastquery.FastQuery(self.connection, self.me) - def get_outputs_filtered(self, owner, include_spent=True): + def get_outputs_filtered(self, owner, spent=None): """ Get a list of output links filtered on some criteria + + Args: + owner (str): base58 encoded public_key. + spent (bool): If ``True`` return only the spent outputs. If + ``False`` return only unspent outputs. If spent is + not specified (``None``) return all outputs. + + Returns: + :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s + pointing to another transaction's condition """ outputs = self.fastquery.get_outputs_by_public_key(owner) - if not include_spent: - outputs = self.fastquery.filter_spent_outputs(outputs) - return outputs + if spent is None: + return outputs + elif spent is True: + return self.fastquery.filter_unspent_outputs(outputs) + elif spent is False: + return self.fastquery.filter_spent_outputs(outputs) def get_transactions_filtered(self, asset_id, operation=None): """ diff --git a/bigchaindb/fastquery.py b/bigchaindb/fastquery.py index d19294ce..985b758e 100644 --- a/bigchaindb/fastquery.py +++ b/bigchaindb/fastquery.py @@ -68,3 +68,18 @@ class FastQuery: for tx in txs for input_ in tx['inputs']} return [ff for ff in outputs if ff not in spends] + + def filter_unspent_outputs(self, outputs): + """ + Remove outputs that have not been spent + + Args: + outputs: list of TransactionLink + """ + links = [o.to_dict() for o in outputs] + res = query.get_spending_transactions(self.connection, links) + txs = [tx for _, tx in self.filter_valid_items(res)] + spends = {TransactionLink.from_dict(input_['fulfills']) + for tx in txs + for input_ in tx['inputs']} + return [ff for ff in outputs if ff in spends] diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index 0fe327bb..945d369c 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -117,6 +117,8 @@ class BlockPipeline: logger.info('Write new block %s with %s transactions', block.id, len(block.transactions)) self.bigchain.write_block(block) + self.bigchain.statsd.incr('pipelines.block.throughput', + len(block.transactions)) return block def delete_tx(self, block): diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index 10b33fd1..3eb6dafa 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -137,9 +137,9 @@ class Vote: self.last_voted_id = block_id del self.counters[block_id] del self.validity[block_id] - return vote + return vote, num_tx - def write_vote(self, vote): + def write_vote(self, vote, num_tx): """Write vote to the database. Args: @@ -149,6 +149,7 @@ class Vote: logger.info("Voting '%s' for block %s", validity, vote['vote']['voting_for_block']) self.bigchain.write_vote(vote) + self.bigchain.statsd.incr('pipelines.vote.throughput', num_tx) return vote diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 0a047344..56bd6259 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -96,8 +96,8 @@ def condition_details_has_owner(condition_details, owner): bool: True if the public key is found in the condition details, False otherwise """ - if 'subfulfillments' in condition_details: - result = condition_details_has_owner(condition_details['subfulfillments'], owner) + if 'subconditions' in condition_details: + result = condition_details_has_owner(condition_details['subconditions'], owner) if result: return True diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 6bf027a0..ee90f2c4 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.11.0.dev' -__short_version__ = '0.11.dev' +__version__ = '1.0.0.dev' +__short_version__ = '1.0.dev' diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 0b24bd64..022320d7 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -61,20 +61,7 @@ def create_app(*, debug=False, threads=1): app = Flask(__name__) - CORS(app, - allow_headers=( - 'x-requested-with', - 'content-type', - 'accept', - 'origin', - 'authorization', - 'x-csrftoken', - 'withcredentials', - 'cache-control', - 'cookie', - 'session-id', - ), - supports_credentials=True) + CORS(app) app.debug = debug diff --git a/bigchaindb/web/views/base.py b/bigchaindb/web/views/base.py index e4ae980b..52cbc0ef 100644 --- a/bigchaindb/web/views/base.py +++ b/bigchaindb/web/views/base.py @@ -3,7 +3,7 @@ Common classes and methods for API handlers """ import logging -from flask import jsonify, request +from flask import jsonify from bigchaindb import config @@ -21,14 +21,9 @@ def make_error(status_code, message=None): return response -def base_url(): - return '%s://%s/' % (request.environ['wsgi.url_scheme'], - request.environ['HTTP_HOST']) - - def base_ws_uri(): """Base websocket uri.""" - # TODO Revisit as this is a workaround to address issue - # https://github.com/bigchaindb/bigchaindb/issues/1465. - host = request.environ['HTTP_HOST'].split(':')[0] - return 'ws://{}:{}'.format(host, config['wsserver']['port']) + scheme = config['wsserver']['scheme'] + host = config['wsserver']['host'] + port = config['wsserver']['port'] + return '{}://{}:{}'.format(scheme, host, port) diff --git a/bigchaindb/web/views/blocks.py b/bigchaindb/web/views/blocks.py index 2471739d..676601c9 100644 --- a/bigchaindb/web/views/blocks.py +++ b/bigchaindb/web/views/blocks.py @@ -41,12 +41,12 @@ class BlockListApi(Resource): "valid", "invalid", "undecided". """ parser = reqparse.RequestParser() - parser.add_argument('tx_id', type=str, required=True) + parser.add_argument('transaction_id', type=str, required=True) parser.add_argument('status', type=str, case_sensitive=False, choices=[Bigchain.BLOCK_VALID, Bigchain.BLOCK_INVALID, Bigchain.BLOCK_UNDECIDED]) args = parser.parse_args(strict=True) - tx_id = args['tx_id'] + tx_id = args['transaction_id'] status = args['status'] pool = current_app.config['bigchain_pool'] diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index 6b01b007..d6240bdb 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -4,7 +4,7 @@ import flask from flask_restful import Resource import bigchaindb -from bigchaindb.web.views.base import base_url, base_ws_uri +from bigchaindb.web.views.base import base_ws_uri from bigchaindb import version from bigchaindb.web.websocket_server import EVENTS_ENDPOINT @@ -15,12 +15,11 @@ class RootIndex(Resource): 'https://docs.bigchaindb.com/projects/server/en/v', version.__version__ + '/' ] - api_v1_url = base_url() + 'api/v1/' return flask.jsonify({ - '_links': { - 'docs': ''.join(docs_url), - 'api_v1': api_v1_url, + 'api': { + 'v1': get_api_v1_info('/api/v1/') }, + 'docs': ''.join(docs_url), 'software': 'BigchainDB', 'version': version.__version__, 'public_key': bigchaindb.config['keypair']['public'], @@ -30,19 +29,26 @@ class RootIndex(Resource): class ApiV1Index(Resource): def get(self): - api_root = base_url() + 'api/v1/' - websocket_root = base_ws_uri() + EVENTS_ENDPOINT - docs_url = [ - 'https://docs.bigchaindb.com/projects/server/en/v', - version.__version__, - '/http-client-server-api.html', - ] - return flask.jsonify({ - '_links': { - 'docs': ''.join(docs_url), - 'self': api_root, - 'statuses': api_root + 'statuses/', - 'transactions': api_root + 'transactions/', - 'streams_v1': websocket_root, - }, - }) + return flask.jsonify(get_api_v1_info('/')) + + +def get_api_v1_info(api_prefix): + """ + Return a dict with all the information specific for the v1 of the + api. + """ + websocket_root = base_ws_uri() + EVENTS_ENDPOINT + docs_url = [ + 'https://docs.bigchaindb.com/projects/server/en/v', + version.__version__, + '/http-client-server-api.html', + ] + + return { + 'docs': ''.join(docs_url), + 'transactions': '{}transactions/'.format(api_prefix), + 'statuses': '{}statuses/'.format(api_prefix), + 'assets': '{}assets/'.format(api_prefix), + 'outputs': '{}outputs/'.format(api_prefix), + 'streams': websocket_root + } diff --git a/bigchaindb/web/views/outputs.py b/bigchaindb/web/views/outputs.py index 735a428f..2f63c07f 100644 --- a/bigchaindb/web/views/outputs.py +++ b/bigchaindb/web/views/outputs.py @@ -15,14 +15,12 @@ class OutputListApi(Resource): parser = reqparse.RequestParser() parser.add_argument('public_key', type=parameters.valid_ed25519, required=True) - parser.add_argument('unspent', type=parameters.valid_bool) - args = parser.parse_args() + parser.add_argument('spent', type=parameters.valid_bool) + args = parser.parse_args(strict=True) pool = current_app.config['bigchain_pool'] - include_spent = not args['unspent'] - with pool() as bigchain: outputs = bigchain.get_outputs_filtered(args['public_key'], - include_spent) - # NOTE: We pass '..' as a path to create a valid relative URI - return [u.to_uri('..') for u in outputs] + args['spent']) + return [{'transaction_id': output.txid, 'output_index': output.output} + for output in outputs] diff --git a/bigchaindb/web/views/statuses.py b/bigchaindb/web/views/statuses.py index a8186146..0044c334 100644 --- a/bigchaindb/web/views/statuses.py +++ b/bigchaindb/web/views/statuses.py @@ -17,44 +17,29 @@ class StatusApi(Resource): ```` is one of "valid", "invalid", "undecided", "backlog". """ parser = reqparse.RequestParser() - parser.add_argument('tx_id', type=str) + parser.add_argument('transaction_id', type=str) parser.add_argument('block_id', type=str) args = parser.parse_args(strict=True) - tx_id = args['tx_id'] + tx_id = args['transaction_id'] block_id = args['block_id'] # logical xor - exactly one query argument required if bool(tx_id) == bool(block_id): - return make_error(400, 'Provide exactly one query parameter. Choices are: block_id, tx_id') + return make_error(400, 'Provide exactly one query parameter. Choices are: block_id, transaction_id') pool = current_app.config['bigchain_pool'] - status, links = None, None + status = None with pool() as bigchain: if tx_id: status = bigchain.get_status(tx_id) - links = { - 'tx': '/transactions/{}'.format(tx_id) - } - elif block_id: _, status = bigchain.get_block(block_id=block_id, include_status=True) - # TODO: enable once blocks endpoint is available - # links = { - # "block": "/blocks/{}".format(args['block_id']) - # } if not status: return make_error(404) - response = { + return { 'status': status } - - if links: - response.update({ - '_links': links - }) - - return response diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index 9f024f54..39c6d529 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -4,7 +4,7 @@ For more information please refer to the documentation: http://bigchaindb.com/ht """ import logging -from flask import current_app, request +from flask import current_app, request, jsonify from flask_restful import Resource, reqparse from bigchaindb.common.exceptions import SchemaValidationError, ValidationError @@ -28,9 +28,9 @@ class TransactionApi(Resource): pool = current_app.config['bigchain_pool'] with pool() as bigchain: - tx = bigchain.get_transaction(tx_id) + tx, status = bigchain.get_transaction(tx_id, include_status=True) - if not tx: + if not tx or status is not bigchain.TX_VALID: return make_error(404) return tx.to_dict() @@ -76,6 +76,7 @@ class TransactionListApi(Resource): ) with pool() as bigchain: + bigchain.statsd.incr('web.tx.post') try: bigchain.validate_transaction(tx_obj) except ValidationError as e: @@ -86,4 +87,16 @@ class TransactionListApi(Resource): else: bigchain.write_transaction(tx_obj) - return tx, 202 + response = jsonify(tx) + response.status_code = 202 + + # NOTE: According to W3C, sending a relative URI is not allowed in the + # Location Header: + # - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + # + # Flask is autocorrecting relative URIs. With the following command, + # we're able to prevent this. + response.autocorrect_location_header = False + status_monitor = '../statuses?transaction_id={}'.format(tx_obj.id) + response.headers['Location'] = status_monitor + return response diff --git a/bigchaindb/web/websocket_server.py b/bigchaindb/web/websocket_server.py index 0aa51ecb..bcf14f35 100644 --- a/bigchaindb/web/websocket_server.py +++ b/bigchaindb/web/websocket_server.py @@ -26,7 +26,7 @@ from bigchaindb.events import EventTypes logger = logging.getLogger(__name__) POISON_PILL = 'POISON_PILL' -EVENTS_ENDPOINT = '/api/v1/streams/valid_tx' +EVENTS_ENDPOINT = '/api/v1/streams/valid_transactions' def _multiprocessing_to_asyncio(in_queue, out_queue, loop): @@ -91,7 +91,7 @@ class Dispatcher: asset_id = tx['id'] if tx['operation'] == 'CREATE' else tx['asset']['id'] data = {'block_id': block['id'], 'asset_id': asset_id, - 'tx_id': tx['id']} + 'transaction_id': tx['id']} str_buffer.append(json.dumps(data)) for _, websocket in self.subscribers.items(): @@ -111,10 +111,15 @@ def websocket_handler(request): while True: # Consume input buffer - msg = yield from websocket.receive() + try: + msg = yield from websocket.receive() + except RuntimeError as e: + logger.debug('Websocket exception: %s', str(e)) + return websocket + if msg.type == aiohttp.WSMsgType.ERROR: logger.debug('Websocket exception: %s', websocket.exception()) - return + return websocket def init_app(event_source, *, loop=None): diff --git a/docker-compose.yml b/docker-compose.yml index c7f3c584..ae988aba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,55 +6,13 @@ services: ports: - "27017" command: mongod --replSet=bigchain-rs - - rdb: - image: rethinkdb - ports: - - "58080:8080" - - "28015" - volumes_from: - - rdb-data - - rdb-2: - image: rethinkdb - ports: - - "8080" - - "29015" - command: rethinkdb --join rdb:29015 --bind all - - rdb-data: - image: rethinkdb:2.3.5 - volumes: - - /data - command: "true" - - bdb-rdb: - build: - context: . - dockerfile: Dockerfile-dev - container_name: docker-bigchaindb - volumes: - - ./bigchaindb:/usr/src/app/bigchaindb - - ./tests:/usr/src/app/tests - - ./docs:/usr/src/app/docs - - ./k8s:/usr/src/app/k8s - - ./setup.py:/usr/src/app/setup.py - - ./setup.cfg:/usr/src/app/setup.cfg - - ./pytest.ini:/usr/src/app/pytest.ini - - ./tox.ini:/usr/src/app/tox.ini - - ./Makefile:/usr/src/app/Makefile - environment: - BIGCHAINDB_DATABASE_BACKEND: rethinkdb - BIGCHAINDB_DATABASE_HOST: rdb - BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 - ports: - - "9984" - command: bigchaindb start bdb: build: context: . dockerfile: Dockerfile-dev + args: + backend: mongodb volumes: - ./bigchaindb:/usr/src/app/bigchaindb - ./tests:/usr/src/app/tests @@ -64,6 +22,7 @@ services: - ./setup.cfg:/usr/src/app/setup.cfg - ./pytest.ini:/usr/src/app/pytest.ini - ./tox.ini:/usr/src/app/tox.ini + - ../cryptoconditions:/usr/src/app/cryptoconditions environment: BIGCHAINDB_DATABASE_BACKEND: mongodb BIGCHAINDB_DATABASE_HOST: mdb diff --git a/docs/root/source/assets.rst b/docs/root/source/assets.rst index 14982406..ff6a17fd 100644 --- a/docs/root/source/assets.rst +++ b/docs/root/source/assets.rst @@ -9,7 +9,7 @@ BigchainDB can store data of any kind (within reason), but it's designed to be p * The owners of an asset can specify (crypto-)conditions which must be satisified by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a transfer transaction. * BigchainDB verifies that the conditions have been satisified as part of checking the validity of transfer transactions. (Moreover, anyone can check that they were satisfied.) * BigchainDB prevents double-spending of an asset. -* Validated transactions are strongly tamper-resistant; see [the section about immutability / tamper-resistance](immutable.html). +* Validated transactions are strongly tamper-resistant; see :doc:`the page about immutability / tamper-resistance `. BigchainDB Integration with Other Blockchains @@ -21,4 +21,4 @@ We’re actively exploring ways that BigchainDB can be used with other blockchai .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or tranfer-enablers. See BigchainDB Server `issue #626 `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See BigchainDB Server `issue #626 `_. diff --git a/docs/root/source/production-ready.md b/docs/root/source/production-ready.md index deb87714..fabae2e4 100644 --- a/docs/root/source/production-ready.md +++ b/docs/root/source/production-ready.md @@ -1,7 +1,14 @@ # Production-Ready? BigchainDB is not production-ready. You can use it to build a prototype or proof-of-concept (POC); many people are already doing that. +Once BigchainDB is production-ready, we'll make an announcement. -BigchainDB Server is currently in version 0.X. ([The Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) has the exact version number.) Once BigchainDB Server is production-ready, we'll issue an announcement. +BigchainDB version numbers follow the conventions of *Semantic Versioning* as documented at [semver.org](http://semver.org/). This means, among other things: + +* Before version 1.0, breaking API changes could happen in any new version, even in a change from version 0.Y.4 to 0.Y.5. + +* Starting with version 1.0.0, breaking API changes will only happen when the MAJOR version changes (e.g. from 1.7.4 to 2.0.0, or from 4.9.3 to 5.0.0). + +To review the release history of some particular BigchainDB software, go to the GitHub repository of that software and click on "Releases". For example, the release history of BigchainDB Server can be found at [https://github.com/bigchaindb/bigchaindb/releases](https://github.com/bigchaindb/bigchaindb/releases). [The BigchainDB Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) will give you a sense of the things we intend to do with BigchainDB in the near term and the long term. \ No newline at end of file diff --git a/docs/root/source/smart-contracts.rst b/docs/root/source/smart-contracts.rst index 0ae1f964..28ae4c6e 100644 --- a/docs/root/source/smart-contracts.rst +++ b/docs/root/source/smart-contracts.rst @@ -7,16 +7,13 @@ BigchainDB will run the subset of smart contracts expressible using "crypto-cond The owners of an asset can impose conditions on it that must be met for the asset to be transferred to new owners. Examples of possible conditions (crypto-conditions) include: -- The current owner must sign the transfer transaction (one which transfers ownership to new owners) -- Three out of five current owners must sign the transfer transaction -- (Shannon and Kelly) or Morgan must sign the transfer transaction -- Anyone who provides the secret password (technically, the preimage of a known hash) can create a valid transfer transaction +- The current owner must sign the transfer transaction (one which transfers ownership to new owners). +- Three out of five current owners must sign the transfer transaction. +- (Shannon and Kelly) or Morgan must sign the transfer transaction. Crypto-conditions can be quite complex if-this-then-that type conditions, where the "this" can be a long boolean expression. Crypto-conditions can't include loops or recursion and are therefore will always run/check in finite time. -BigchainDB also supports a timeout condition which enables it to support a form of escrow. - .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or tranfer-enablers. See BigchainDB Server `issue #626 `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See BigchainDB Server `issue #626 `_. \ No newline at end of file diff --git a/docs/root/source/transaction-concepts.md b/docs/root/source/transaction-concepts.md index 58d8a013..66c4ec0f 100644 --- a/docs/root/source/transaction-concepts.md +++ b/docs/root/source/transaction-concepts.md @@ -1,11 +1,12 @@ # Transaction Concepts -In BigchainDB, _Transactions_ are used to register, issue, create or transfer +In BigchainDB, _transactions_ are used to register, issue, create or transfer things (e.g. assets). Transactions are the most basic kind of record stored by BigchainDB. There are two kinds: CREATE transactions and TRANSFER transactions. + ## CREATE Transactions A CREATE transaction can be used to register, issue, create or otherwise @@ -14,29 +15,57 @@ one might register an identity or a creative work. The things are often called "assets" but they might not be literal assets. BigchainDB supports divisible assets as of BigchainDB Server v0.8.0. -That means you can create/register an asset with an initial quantity, -e.g. 700 oak trees. Divisible assets can be split apart or recombined -by transfer transactions (described more below). +That means you can create/register an asset with an initial number of "shares." +For example, A CREATE transaction could register a truckload of 50 oak trees. +Each share of a divisible asset must be interchangeable with each other share; +the shares must be fungible. + +A CREATE transaction can have one or more outputs. +Each output has an associated amount: the number of shares tied to that output. +For example, if the asset consists of 50 oak trees, +one output might have 35 oak trees for one set of owners, +and the other output might have 15 oak trees for another set of owners. + +Each output also has an associated condition: the condition that must be met +(by a TRANSFER transaction) to transfer/spend the output. +BigchainDB supports a variety of conditions, +a subset of the [Interledger Protocol (ILP)](https://interledger.org/) +crypto-conditions. For details, see +[the documentation about Inputs and Outputs](https://docs.bigchaindb.com/projects/server/en/latest/data-models/inputs-outputs.html). + +Each output also has a list of all the public keys associated +with the conditions on that output. +Loosely speaking, that list might be interpreted as the list of "owners." +A more accurate word might be fulfillers, signers, controllers, +or transfer-enablers. +See BigchainDB Server [issue #626](https://github.com/bigchaindb/bigchaindb/issues/626). + +A CREATE transaction must be signed by all the owners. +(If you're looking for that signature, +it's in the one "fulfillment" of the one input, albeit encoded.) -A CREATE transaction also establishes, in its outputs, the conditions that must -be met to transfer the asset(s). The conditions may also be associated with a -list of public keys that, depending on the condition, may have full or partial -control over the asset(s). For example, there may be a condition that any -transfer must be signed (cryptographically) by the private key associated with a -given public key. More sophisticated conditions are possible. BigchainDB's -conditions are based on the crypto-conditions of the [Interledger Protocol -(ILP)](https://interledger.org/). ## TRANSFER Transactions -A TRANSFER transaction can transfer an asset -by providing inputs which fulfill the current output conditions on the asset. -It must also specify new transfer conditions. +A TRANSFER transaction can transfer/spend one or more outputs +on other transactions (CREATE transactions or other TRANSFER transactions). +Those outputs must all be associated with the same asset; +a TRANSFER transaction can only transfer shares of one asset at a time. + +Each input on a TRANSFER transaction connects to one output +on another transaction. +Each input must satisfy the condition on the output it's trying +to transfer/spend. + +A TRANSFER transaction can have one or more outputs, +just like a CREATE transaction (described above). +The total number of shares coming in on the inputs must equal +the total number of shares going out on the outputs. **Example 1:** Suppose a red car is owned and controlled by Joe. Suppose the current transfer condition on the car says that any valid transfer must be signed by Joe. -Joe and a buyer named Rae could build a TRANSFER transaction containing +Joe could build a TRANSFER transaction containing an input with Joe's signature (to fulfill the current output condition) plus a new output condition saying that any valid transfer must be signed by Rae. @@ -53,33 +82,18 @@ transferred if both Jack and Kelly sign. Note how the sum of the incoming paperclips must equal the sum of the outgoing paperclips (100). + ## Transaction Validity When a node is asked to check if a transaction is valid, it checks several -things. Some things it checks are: +things. We documented those things in a post on *The BigchainDB Blog*: +["What is a Valid Transaction in BigchainDB?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598) +(Note: That post was about BigchainDB Server v1.0.0.) -* Are all the fulfillments valid? (Do they correctly satisfy the conditions - they claim to satisfy?) -* If it's a creation transaction, is the asset valid? -* If it's a transfer transaction: - * Is it trying to fulfill a condition in a nonexistent transaction? - * Is it trying to fulfill a condition that's not in a valid transaction? - (It's okay if the condition is in a transaction in an invalid block; those - transactions are ignored. Transactions in the backlog or undecided blocks - are not ignored.) - * Is it trying to fulfill a condition that has already been fulfilled, or - that some other pending transaction (in the backlog or an undecided block) - also aims to fulfill? - * Is the asset ID in the transaction the same as the asset ID in all - transactions whose conditions are being fulfilled? - * Is the sum of the amounts in the fulfillments equal - to the sum of the amounts in the new conditions? -If you're curious about the details of transaction validation, the code is in -the `validate` method of the `Transaction` class, in `bigchaindb/models.py` (at -the time of writing). +## Example Transactions -Note: The check to see if the transaction ID is equal to the hash of the -transaction body is actually done whenever the transaction is converted from a -Python dict to a Transaction object, which must be done before the `validate` -method can be called (since it's called on a Transaction object). +There are example BigchainDB transactions in +[the HTTP API documentation](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html) +and +[the Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html). diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 731bee2c..276c7f44 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -68,6 +68,7 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ HTTP/1.1 202 Accepted +Location: ../statuses?transaction_id=%(txid)s Content-Type: application/json %(tx)s @@ -75,7 +76,7 @@ Content-Type: application/json TPLS['get-statuses-tx-request'] = """\ -GET /statuses?tx_id=%(txid)s HTTP/1.1 +GET /statuses?transaction_id=%(txid)s HTTP/1.1 Host: example.com """ @@ -96,10 +97,7 @@ HTTP/1.1 200 OK Content-Type: application/json { - "status": "valid", - "_links": { - "tx": "/transactions/%(txid)s" - } + "status": "valid" } """ @@ -126,10 +124,7 @@ HTTP/1.1 200 OK Content-Type: application/json { - "status": "valid", - "_links": { - "block": "/blocks/%(blockid)s" - } + "status": "valid" } """ @@ -150,7 +145,7 @@ Content-Type: application/json TPLS['get-block-txid-request'] = """\ -GET /api/v1/blocks?tx_id=%(txid)s HTTP/1.1 +GET /api/v1/blocks?transaction_id=%(txid)s HTTP/1.1 Host: example.com """ diff --git a/docs/server/source/_static/mongodb_cloud_manager_1.png b/docs/server/source/_static/mongodb_cloud_manager_1.png new file mode 100644 index 00000000..16073d6b Binary files /dev/null and b/docs/server/source/_static/mongodb_cloud_manager_1.png differ diff --git a/docs/server/source/clusters-feds/aws-testing-cluster.md b/docs/server/source/appendices/aws-testing-cluster.md similarity index 100% rename from docs/server/source/clusters-feds/aws-testing-cluster.md rename to docs/server/source/appendices/aws-testing-cluster.md diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index f5931e64..25eb7e19 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -18,6 +18,7 @@ Appendices backend commands aws-setup + aws-testing-cluster template-terraform-aws template-ansible azure-quickstart-template diff --git a/docs/server/source/appendices/install-latest-pip.md b/docs/server/source/appendices/install-latest-pip.md index 97405882..fc992fee 100644 --- a/docs/server/source/appendices/install-latest-pip.md +++ b/docs/server/source/appendices/install-latest-pip.md @@ -5,7 +5,7 @@ You can check the version of `pip` you're using (in your current virtualenv) by pip -V ``` -If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.4, then you must install a `pip` version associated with Python 3.4+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). +If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.5, then you must install a `pip` version associated with Python 3.5+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). On Ubuntu 16.04, we found that this works: ```text @@ -17,4 +17,4 @@ That should install a Python 3 version of `pip` named `pip3`. If that didn't wor You can upgrade `pip` (`pip3`) and `setuptools` to the latest versions using: ```text pip3 install --upgrade pip setuptools -``` \ No newline at end of file +``` diff --git a/docs/server/source/cloud-deployment-templates/first-node.rst b/docs/server/source/cloud-deployment-templates/first-node.rst deleted file mode 100644 index 298d7ceb..00000000 --- a/docs/server/source/cloud-deployment-templates/first-node.rst +++ /dev/null @@ -1,463 +0,0 @@ -First Node or Bootstrap Node Setup -================================== - -This document is a work in progress and will evolve over time to include -security, websocket and other settings. - - -Step 1: Set Up the Cluster --------------------------- - - .. code:: bash - - az group create --name bdb-test-cluster-0 --location westeurope --debug --output json - - ssh-keygen -t rsa -C "k8s-bdb-test-cluster-0" -f ~/.ssh/k8s-bdb-test-cluster-0 - - az acs create --name k8s-bdb-test-cluster-0 \ - --resource-group bdb-test-cluster-0 \ - --master-count 3 \ - --agent-count 2 \ - --admin-username ubuntu \ - --agent-vm-size Standard_D2_v2 \ - --dns-prefix k8s-bdb-test-cluster-0 \ - --ssh-key-value ~/.ssh/k8s-bdb-test-cluster-0.pub \ - --orchestrator-type kubernetes \ - --debug --output json - - az acs kubernetes get-credentials \ - --resource-group bdb-test-cluster-0 \ - --name k8s-bdb-test-cluster-0 \ - --debug --output json - - echo -e "Host k8s-bdb-test-cluster-0.westeurope.cloudapp.azure.com\n ForwardAgent yes" >> ~/.ssh/config - - -Step 2: Connect to the Cluster UI - (optional) ----------------------------------------------- - - * Get the kubectl context for this cluster using ``kubectl config view``. - - * For the above commands, the context would be ``k8s-bdb-test-cluster-0``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 proxy -p 8001 - -Step 3. Configure the Cluster ------------------------------ - - * Use the ConfigMap in ``configuration/config-map.yaml`` file for configuring - the cluster. - - * Log in the the MongoDB Cloud Manager and select the group that will monitor - and backup this cluster from the dropdown box. - - * Go to Settings, Group Settings and copy the ``Agent Api Key``. - - * Replace the ```` field with this key. - - * Since this is the first node of the cluster, ensure that the ``data.fqdn`` - field has the value ``mdb-instance-0``. - - * We only support the value ``all`` in the ``data.allowed-hosts`` field for now. - - * Create the ConfigMap - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f configuration/config-map.yaml - -Step 4. Start the NGINX Service -------------------------------- - - * This will will give us a public IP for the cluster. - - * Once you complete this step, you might need to wait up to 10 mins for the - public IP to be assigned. - - * You have the option to use vanilla NGINX or an OpenResty NGINX integrated - with 3scale API Gateway. - - -Step 4.1. Vanilla NGINX -^^^^^^^^^^^^^^^^^^^^^^^ - - * This configuration is located in the file ``nginx/nginx-svc.yaml``. - - * Since this is the first node, rename ``metadata.name`` and ``metadata.labels.name`` - to ``ngx-instance-0``, and ``spec.selector.app`` to ``ngx-instance-0-dep``. - - * Start the Kubernetes Service: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f nginx/nginx-svc.yaml - - -Step 4.2. OpenResty NGINX + 3scale -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - * You have to enable HTTPS for this one and will need an HTTPS certificate - for your domain - - * Assuming that the public key chain is named ``cert.pem`` and private key is - ``cert.key``, run the following commands to encode the certificates into - single continuous string that can be embedded in yaml. - - .. code:: bash - - cat cert.pem | base64 -w 0 > cert.pem.b64 - - cat cert.key | base64 -w 0 > cert.key.b64 - - - * Copy the contents of ``cert.pem.b64`` in the ``cert.pem`` field, and the - contents of ``cert.key.b64`` in the ``cert.key`` field in the file - ``nginx-3scale/nginx-3scale-secret.yaml`` - - * Create the Kubernetes Secret: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-secret.yaml - - * Since this is the first node, rename ``metadata.name`` and ``metadata.labels.name`` - to ``ngx-instance-0``, and ``spec.selector.app`` to ``ngx-instance-0-dep`` in - ``nginx-3scale/nginx-3scale-svc.yaml`` file. - - * Start the Kubernetes Service: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-svc.yaml - - -Step 5. Assign DNS Name to the NGINX Public IP ----------------------------------------------- - - * The following command can help you find out if the nginx service strated above - has been assigned a public IP or external IP address: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 get svc -w - - * Once a public IP is assigned, you can log in to the Azure portal and map it to - a DNS name. - - * We usually start with bdb-test-cluster-0, bdb-test-cluster-1 and so on. - - * Let us assume that we assigned the unique name of ``bdb-test-cluster-0`` here. - - -Step 6. Start the Mongo Kubernetes Service ------------------------------------------- - - * Change ``metadata.name`` and ``metadata.labels.name`` to - ``mdb-instance-0``, and ``spec.selector.app`` to ``mdb-instance-0-ss``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-svc.yaml - - -Step 7. Start the BigchainDB Kubernetes Service ------------------------------------------------ - - * Change ``metadata.name`` and ``metadata.labels.name`` to - ``bdb-instance-0``, and ``spec.selector.app`` to ``bdb-instance-0-dep``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f bigchaindb/bigchaindb-svc.yaml - - -Step 8. Start the NGINX Kubernetes Deployment ---------------------------------------------- - - * As in step 4, you have the option to use vanilla NGINX or an OpenResty NGINX - integrated with 3scale API Gateway. - -Step 8.1. Vanilla NGINX -^^^^^^^^^^^^^^^^^^^^^^^ - - * This configuration is located in the file ``nginx/nginx-dep.yaml``. - - * Since this is the first node, change the ``metadata.name`` and - ``spec.template.metadata.labels.app`` to ``ngx-instance-0-dep``. - - * Set ``MONGODB_BACKEND_HOST`` env var to - ``mdb-instance-0.default.svc.cluster.local``. - - * Set ``BIGCHAINDB_BACKEND_HOST`` env var to - ``bdb-instance-0.default.svc.cluster.local``. - - * Set ``MONGODB_FRONTEND_PORT`` to - ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_MDB_PORT)``. - - * Set ``BIGCHAINDB_FRONTEND_PORT`` to - ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT)``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f nginx/nginx-dep.yaml - -Step 8.2. OpenResty NGINX + 3scale -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - * This configuration is located in the file - ``nginx-3scale/nginx-3scale-dep.yaml``. - - * Since this is the first node, change the metadata.name and - spec.template.metadata.labels.app to ``ngx-instance-0-dep``. - - * Set ``MONGODB_BACKEND_HOST`` env var to - ``mdb-instance-0.default.svc.cluster.local``. - - * Set ``BIGCHAINDB_BACKEND_HOST`` env var to - ``bdb-instance-0.default.svc.cluster.local``. - - * Set ``MONGODB_FRONTEND_PORT`` to - ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_MDB_PORT)``. - - * Set ``BIGCHAINDB_FRONTEND_PORT`` to - ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT)``. - - * Also, replace the placeholder strings for the env vars with the values - obtained from 3scale. You will need the Secret Token, Service ID, Version Header - and Provider Key from 3scale. - - * The ``THREESCALE_FRONTEND_API_DNS_NAME`` will be DNS name registered for your - HTTPS certificate. - - * You can set the ``THREESCALE_UPSTREAM_API_PORT`` to any port other than 9984, - 9985, 443, 8888 and 27017. We usually use port ``9999``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-dep.yaml - - -Step 9. Create a Kubernetes Storage Class for MongoDB ------------------------------------------------------ - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-sc.yaml - - -Step 10. Create a Kubernetes PersistentVolumeClaim --------------------------------------------------- - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-pvc.yaml - - -Step 11. Start a Kubernetes StatefulSet for MongoDB ---------------------------------------------------- - - * Change ``spec.serviceName`` to ``mdb-instance-0``. - - * Change the ``metadata.name``, ``template.metadata.name`` and - ``template.metadata.labels.app`` to ``mdb-instance-0-ss``. - - * It might take up to 10 minutes for the disks to be created and attached to - the pod. - - * The UI might show that the pod has errored with the - message "timeout expired waiting for volumes to attach/mount". - - * Use the CLI below to check the status of the pod in this case, - instead of the UI. This happens due to a bug in Azure ACS. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-ss.yaml - - * You can check the status of the pod using the command: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 get po -w - - -Step 12. Start a Kubernetes Deployment for Bigchaindb ------------------------------------------------------ - - * Change both ``metadata.name`` and ``spec.template.metadata.labels.app`` - to ``bdb-instance-0-dep``. - - * Set ``BIGCHAINDB_DATABASE_HOST`` to ``mdb-instance-0``. - - * Set the appropriate ``BIGCHAINDB_KEYPAIR_PUBLIC``, - ``BIGCHAINDB_KEYPAIR_PRIVATE`` values. - - * One way to generate BigchainDB keypair is to run a Python shell with - the command - ``from bigchaindb_driver import crypto; crypto.generate_keypair()``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f bigchaindb/bigchaindb-dep.yaml - - -Step 13. Start a Kubernetes Deployment for MongoDB Monitoring Agent -------------------------------------------------------------------- - - * Change both metadata.name and spec.template.metadata.labels.app to - ``mdb-mon-instance-0-dep``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb-monitoring-agent/mongo-mon-dep.yaml - - * Get the pod name and check its logs: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 get po - - kubectl --context k8s-bdb-test-cluster-0 logs -f - - -Step 14. Configure MongoDB Cloud Manager for Monitoring -------------------------------------------------------- - - * Open `MongoDB Cloud Manager `_. - - * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud Manager. - - * Select the group from the dropdown box on the page. - - * Go to Settings, Group Settings and add a Preferred Hostnames regexp as - ``^mdb-instance-[0-9]{1,2}$``. It may take up to 5 mins till this setting - is in effect. You may refresh the browser window and verify whether the changes - have been saved or not. - - * Next, click the ``Deployment`` tab, and then the ``Manage Existing`` button. - - * On the ``Import your deployment for monitoring`` page, enter the hostname as - ``mdb-instance-0``, port number as ``27017``, with no authentication and no - TLS/SSL settings. - - * Once the deployment is found, click the ``Continue`` button. - This may take about a minute or two. - - * Do not add ``Automation Agent`` when given an option to add it. - - * Verify on the UI that data is being by the monitoring agent. - - -Step 15. Start a Kubernetes Deployment for MongoDB Backup Agent ---------------------------------------------------------------- - - * Change both ``metadata.name`` and ``spec.template.metadata.labels.app`` - to ``mdb-backup-instance-0-dep``. - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb-backup-agent/mongo-backup-dep.yaml - - * Get the pod name and check its logs: - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 get po - - kubectl --context k8s-bdb-test-cluster-0 logs -f - - -Step 16. Configure MongoDB Cloud Manager for Backup ---------------------------------------------------- - - * Open `MongoDB Cloud Manager `_. - - * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud - Manager. - - * Select the group from the dropdown box on the page. - - * Click ``Backup`` tab. - - * Click on the ``Begin Setup``. - - * Click on ``Next``, select the replica set from the dropdown menu. - - * Verify the details of your MongoDB instance and click on ``Start`` again. - - * It might take up to 5 minutes to start the backup process. - - * Verify that data is being backed up on the UI. - - -Step 17. Verify that the Cluster is Correctly Set Up ----------------------------------------------------- - - * Start the toolbox container in the cluster - - .. code:: bash - - kubectl --context k8s-bdb-test-cluster-0 \ - run -it toolbox \ - --image bigchaindb/toolbox \ - --image-pull-policy=Always \ - --restart=Never --rm - - * Verify MongoDB instance - - .. code:: bash - - nslookup mdb-instance-0 - - dig +noall +answer _mdb-port._tcp.mdb-instance-0.default.svc.cluster.local SRV - - curl -X GET http://mdb-instance-0:27017 - - * Verify BigchainDB instance - - .. code:: bash - - nslookup bdb-instance-0 - - dig +noall +answer _bdb-port._tcp.bdb-instance-0.default.svc.cluster.local SRV - - dig +noall +answer _bdb-ws-port._tcp.bdb-instance-0.default.svc.cluster.local SRV - - curl -X GET http://bdb-instance-0:9984 - - wsc ws://bdb-instance-0:9985/api/v1/streams/valid_tx - - * Verify NGINX instance - - .. code:: bash - - nslookup ngx-instance-0 - - dig +noall +answer _ngx-public-mdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV - - curl -X GET http://ngx-instance-0:27017 # results in curl: (56) Recv failure: Connection reset by peer - - dig +noall +answer _ngx-public-bdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV - - dig +noall +answer _ngx-public-ws-port._tcp.ngx-instance-0.default.svc.cluster.local SRV - - * If you have run the vanilla NGINX instance, run - - .. code:: bash - - curl -X GET http://ngx-instance-0:80 - - wsc ws://ngx-instance-0:81/api/v1/streams/valid_tx - - * If you have the OpenResty NGINX + 3scale instance, run - - .. code:: bash - - curl -X GET https://ngx-instance-0 - - * Check the MongoDB monitoring and backup agent on the MongoDB Coud Manager portal to verify they are working fine. - - * Send some transactions to BigchainDB and verify it's up and running! - diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst deleted file mode 100644 index 8c38e384..00000000 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ /dev/null @@ -1,476 +0,0 @@ -Kubernetes Template: Deploy a Single BigchainDB Node -==================================================== - -This page describes how to deploy the first BigchainDB node -in a BigchainDB cluster, or a stand-alone BigchainDB node, -using `Kubernetes `_. -It assumes you already have a running Kubernetes cluster. - -If you want to add a new BigchainDB node to an existing BigchainDB cluster, -refer to :doc:`the page about that `. - - -Step 1: Install kubectl ------------------------ - -kubectl is the Kubernetes CLI. -If you don't already have it installed, -then see the `Kubernetes docs to install it -`_. - - -Step 2: Configure kubectl -------------------------- - -The default location of the kubectl configuration file is ``~/.kube/config``. -If you don't have that file, then you need to get it. - -**Azure.** If you deployed your Kubernetes cluster on Azure -using the Azure CLI 2.0 (as per :doc:`our template `), -then you can get the ``~/.kube/config`` file using: - -.. code:: bash - - $ az acs kubernetes get-credentials \ - --resource-group \ - --name - -If it asks for a password (to unlock the SSH key) -and you enter the correct password, -but you get an error message, -then try adding ``--ssh-key-file ~/.ssh/`` -to the above command (i.e. the path to the private key). - - -Step 3: Create Storage Classes ------------------------------- - -MongoDB needs somewhere to store its data persistently, -outside the container where MongoDB is running. -Our MongoDB Docker container -(based on the official MongoDB Docker container) -exports two volume mounts with correct -permissions from inside the container: - -* The directory where the mongod instance stores its data: ``/data/db``. - There's more explanation in the MongoDB docs about `storage.dbpath `_. - -* The directory where the mongodb instance stores the metadata for a sharded - cluster: ``/data/configdb/``. - There's more explanation in the MongoDB docs about `sharding.configDB `_. - -Explaining how Kubernetes handles persistent volumes, -and the associated terminology, -is beyond the scope of this documentation; -see `the Kubernetes docs about persistent volumes -`_. - -The first thing to do is create the Kubernetes storage classes. - -**Azure.** First, you need an Azure storage account. -If you deployed your Kubernetes cluster on Azure -using the Azure CLI 2.0 -(as per :doc:`our template `), -then the `az acs create` command already created two -storage accounts in the same location and resource group -as your Kubernetes cluster. -Both should have the same "storage account SKU": ``Standard_LRS``. -Standard storage is lower-cost and lower-performance. -It uses hard disk drives (HDD). -LRS means locally-redundant storage: three replicas -in the same data center. -Premium storage is higher-cost and higher-performance. -It uses solid state drives (SSD). -At the time of writing, -when we created a storage account with SKU ``Premium_LRS`` -and tried to use that, -the PersistentVolumeClaim would get stuck in a "Pending" state. -For future reference, the command to create a storage account is -`az storage account create `_. - - -Get the file ``mongo-sc.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-sc.yaml - -You may have to update the ``parameters.location`` field in both the files to -specify the location you are using in Azure. - -Create the required storage classes using: - -.. code:: bash - - $ kubectl apply -f mongo-sc.yaml - - -You can check if it worked using ``kubectl get storageclasses``. - -**Azure.** Note that there is no line of the form -``storageAccount: `` -under ``parameters:``. When we included one -and then created a PersistentVolumeClaim based on it, -the PersistentVolumeClaim would get stuck -in a "Pending" state. -Kubernetes just looks for a storageAccount -with the specified skuName and location. - - -Step 4: Create Persistent Volume Claims ---------------------------------------- - -Next, you will create two PersistentVolumeClaim objects ``mongo-db-claim`` and -``mongo-configdb-claim``. -Get the file ``mongo-pvc.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-pvc.yaml - -Note how there's no explicit mention of Azure, AWS or whatever. -``ReadWriteOnce`` (RWO) means the volume can be mounted as -read-write by a single Kubernetes node. -(``ReadWriteOnce`` is the *only* access mode supported -by AzureDisk.) -``storage: 20Gi`` means the volume has a size of 20 -`gibibytes `_. - -You may want to update the ``spec.resources.requests.storage`` field in both -the files to specify a different disk size. - -Create the required Persistent Volume Claims using: - -.. code:: bash - - $ kubectl apply -f mongo-pvc.yaml - - -You can check its status using: ``kubectl get pvc -w`` - -Initially, the status of persistent volume claims might be "Pending" -but it should become "Bound" fairly quickly. - - -Step 5: Create the Config Map - Optional ----------------------------------------- - -This step is required only if you are planning to set up multiple -`BigchainDB nodes -`_. - -MongoDB reads the local ``/etc/hosts`` file while bootstrapping a replica set -to resolve the hostname provided to the ``rs.initiate()`` command. It needs to -ensure that the replica set is being initialized in the same instance where -the MongoDB instance is running. - -To achieve this, you will create a ConfigMap with the FQDN of the MongoDB instance -and populate the ``/etc/hosts`` file with this value so that a replica set can -be created seamlessly. - -Get the file ``mongo-cm.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-cm.yaml - -You may want to update the ``data.fqdn`` field in the file before creating the -ConfigMap. ``data.fqdn`` field will be the DNS name of your MongoDB instance. -This will be used by other MongoDB instances when forming a MongoDB -replica set. It should resolve to the MongoDB instance in your cluster when -you are done with the setup. This will help when you are adding more MongoDB -instances to the replica set in the future. - - -**Azure.** -In Kubernetes on ACS, the name you populate in the ``data.fqdn`` field -will be used to configure a DNS name for the public IP assigned to the -Kubernetes Service that is the frontend for the MongoDB instance. -We suggest using a name that will already be available in Azure. -We use ``mdb-instance-0``, ``mdb-instance-1`` and so on in this document, -which gives us ``mdb-instance-0..cloudapp.azure.com``, -``mdb-instance-1..cloudapp.azure.com``, etc. as the FQDNs. -The ```` is the Azure datacenter location you are using, -which can also be obtained using the ``az account list-locations`` command. -You can also try to assign a name to an Public IP in Azure before starting -the process, or use ``nslookup`` with the name you have in mind to check -if it's available for use. - -You should ensure that the the name specified in the ``data.fqdn`` field is -a unique one. - -**Kubernetes on bare-metal or other cloud providers.** -You need to provide the name resolution function -by other means (using DNS providers like GoDaddy, CloudFlare or your own -private DNS server). The DNS set up for other environments is currently -beyond the scope of this document. - - -Create the required ConfigMap using: - -.. code:: bash - - $ kubectl apply -f mongo-cm.yaml - - -You can check its status using: ``kubectl get cm`` - -Now you are ready to run MongoDB and BigchainDB on our Kubernetes cluster. - - -Step 6: Run MongoDB as a StatefulSet ------------------------------------- - -Get the file ``mongo-ss.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/mongodb/mongo-ss.yaml - - -Note how the MongoDB container uses the ``mongo-db-claim`` and the -``mongo-configdb-claim`` PersistentVolumeClaims for its ``/data/db`` and -``/data/configdb`` diretories (mount path). Note also that we use the pod's -``securityContext.capabilities.add`` specification to add the ``FOWNER`` -capability to the container. -That is because MongoDB container has the user ``mongodb``, with uid ``999`` -and group ``mongodb``, with gid ``999``. -When this container runs on a host with a mounted disk, the writes fail when -there is no user with uid ``999``. -To avoid this, we use the Docker feature of ``--cap-add=FOWNER``. -This bypasses the uid and gid permission checks during writes and allows data -to be persisted to disk. -Refer to the -`Docker docs `_ -for details. - -As we gain more experience running MongoDB in testing and production, we will -tweak the ``resources.limits.cpu`` and ``resources.limits.memory``. -We will also stop exposing port ``27017`` globally and/or allow only certain -hosts to connect to the MongoDB instance in the future. - -Create the required StatefulSet using: - -.. code:: bash - - $ kubectl apply -f mongo-ss.yaml - -You can check its status using the commands ``kubectl get statefulsets -w`` -and ``kubectl get svc -w`` - -You may have to wait for up to 10 minutes for the disk to be created -and attached on the first run. The pod can fail several times with the message -saying that the timeout for mounting the disk was exceeded. - - -Step 7: Initialize a MongoDB Replica Set - Optional ---------------------------------------------------- - -This step is required only if you are planning to set up multiple -`BigchainDB nodes -`_. - - -Login to the running MongoDB instance and access the mongo shell using: - -.. code:: bash - - $ kubectl exec -it mdb-0 -c mongodb -- /bin/bash - root@mdb-0:/# mongo --port 27017 - -You will initiate the replica set by using the ``rs.initiate()`` command from the -mongo shell. Its syntax is: - -.. code:: bash - - rs.initiate({ - _id : ":" - } ] - }) - -An example command might look like: - -.. code:: bash - - > rs.initiate({ _id : "bigchain-rs", members: [ { _id : 0, host :"mdb-instance-0.westeurope.cloudapp.azure.com:27017" } ] }) - - -where ``mdb-instance-0.westeurope.cloudapp.azure.com`` is the value stored in -the ``data.fqdn`` field in the ConfigMap created using ``mongo-cm.yaml``. - - -You should see changes in the mongo shell prompt from ``>`` -to ``bigchain-rs:OTHER>`` to ``bigchain-rs:SECONDARY>`` and finally -to ``bigchain-rs:PRIMARY>``. - -You can use the ``rs.conf()`` and the ``rs.status()`` commands to check the -detailed replica set configuration now. - - -Step 8: Create a DNS record - Optional --------------------------------------- - -This step is required only if you are planning to set up multiple -`BigchainDB nodes -`_. - -**Azure.** Select the current Azure resource group and look for the ``Public IP`` -resource. You should see at least 2 entries there - one for the Kubernetes -master and the other for the MongoDB instance. You may have to ``Refresh`` the -Azure web page listing the resources in a resource group for the latest -changes to be reflected. -Select the ``Public IP`` resource that is attached to your service (it should -have the Kubernetes cluster name along with a random string), -select ``Configuration``, add the DNS name that was added in the -ConfigMap earlier, click ``Save``, and wait for the changes to be applied. - -To verify the DNS setting is operational, you can run ``nslookup `` from your local Linux shell. - -This will ensure that when you scale the replica set later, other MongoDB -members in the replica set can reach this instance. - - -Step 9: Run BigchainDB as a Deployment --------------------------------------- - -Get the file ``bigchaindb-dep.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/bigchaindb/bigchaindb-dep.yaml - -Note that we set the ``BIGCHAINDB_DATABASE_HOST`` to ``mdb-svc`` which is the -name of the MongoDB service defined earlier. - -We also hardcode the ``BIGCHAINDB_KEYPAIR_PUBLIC``, -``BIGCHAINDB_KEYPAIR_PRIVATE`` and ``BIGCHAINDB_KEYRING`` for now. - -As we gain more experience running BigchainDB in testing and production, we -will tweak the ``resources.limits`` values for CPU and memory, and as richer -monitoring and probing becomes available in BigchainDB, we will tweak the -``livenessProbe`` and ``readinessProbe`` parameters. - -We also plan to specify scheduling policies for the BigchainDB deployment so -that we ensure that BigchainDB and MongoDB are running in separate nodes, and -build security around the globally exposed port ``9984``. - -Create the required Deployment using: - -.. code:: bash - - $ kubectl apply -f bigchaindb-dep.yaml - -You can check its status using the command ``kubectl get deploy -w`` - - -Step 10: Run NGINX as a Deployment ----------------------------------- - -NGINX is used as a proxy to both the BigchainDB and MongoDB instances in the -node. -It proxies HTTP requests on port 80 to the BigchainDB backend, and TCP -connections on port 27017 to the MongoDB backend. - -You can also configure a whitelist in NGINX to allow only connections from -other instances in the MongoDB replica set to access the backend MongoDB -instance. - -Get the file ``nginx-cm.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/nginx/nginx-cm.yaml - -The IP address whitelist can be explicitly configured in ``nginx-cm.yaml`` -file. You will need a list of the IP addresses of all the other MongoDB -instances in the cluster. If the MongoDB intances specify a hostname, then this -needs to be resolved to the corresponding IP addresses. If the IP address of -any MongoDB instance changes, we can start a 'rolling upgrade' of NGINX after -updating the corresponding ConfigMap without affecting availabilty. - - -Create the ConfigMap for the whitelist using: - -.. code:: bash - - $ kubectl apply -f nginx-cm.yaml - -Get the file ``nginx-dep.yaml`` from GitHub using: - -.. code:: bash - - $ wget https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/k8s/nginx/nginx-dep.yaml - -Create the NGINX deployment using: - -.. code:: bash - - $ kubectl apply -f nginx-dep.yaml - - -Step 11: Verify the BigchainDB Node Setup ------------------------------------------ - -Step 11.1: Testing Internally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Run a container that provides utilities like ``nslookup``, ``curl`` and ``dig`` -on the cluster and query the internal DNS and IP endpoints. - -.. code:: bash - - $ kubectl run -it toolbox -- image --restart=Never --rm - -There is a generic image based on alpine:3.5 with the required utilities -hosted at Docker Hub under `bigchaindb/toolbox `_. -The corresponding Dockerfile is in the bigchaindb/bigchaindb repository on GitHub, at `https://github.com/bigchaindb/bigchaindb/blob/master/k8s/toolbox/Dockerfile `_. - -You can use it as below to get started immediately: - -.. code:: bash - - $ kubectl run -it toolbox --image bigchaindb/toolbox --restart=Never --rm - -It will drop you to the shell prompt. -Now you can query for the ``mdb`` and ``bdb`` service details. - -.. code:: bash - - # nslookup mdb-svc - # nslookup bdb-svc - # nslookup ngx-svc - # dig +noall +answer _mdb-port._tcp.mdb-svc.default.svc.cluster.local SRV - # dig +noall +answer _bdb-port._tcp.bdb-svc.default.svc.cluster.local SRV - # dig +noall +answer _ngx-public-mdb-port._tcp.ngx-svc.default.svc.cluster.local SRV - # dig +noall +answer _ngx-public-bdb-port._tcp.ngx-svc.default.svc.cluster.local SRV - # curl -X GET http://mdb-svc:27017 - # curl -X GET http://bdb-svc:9984 - # curl -X GET http://ngx-svc:80 - # curl -X GET http://ngx-svc:27017 - -The ``nslookup`` commands should output the configured IP addresses of the -services in the cluster - -The ``dig`` commands should return the port numbers configured for the -various services in the cluster. - -Finally, the ``curl`` commands test the availability of the services -themselves. - -Step 11.2: Testing Externally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Try to access the ``:80`` -on your browser. You must receive a json output that shows the BigchainDB -server version among other things. - -Try to access the ``:27017`` -on your browser. If your IP is in the whitelist, you will receive a message -from the MongoDB instance stating that it doesn't allow HTTP connections to -the port anymore. If your IP is not in the whitelist, your access will be -blocked and you will not see any response from the MongoDB instance. - diff --git a/docs/server/source/clusters-feds/index.rst b/docs/server/source/clusters-feds/index.rst deleted file mode 100644 index 40e3b873..00000000 --- a/docs/server/source/clusters-feds/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Clusters -======== - -.. toctree:: - :maxdepth: 1 - - set-up-a-cluster - aws-testing-cluster - diff --git a/docs/server/source/clusters-feds/set-up-a-cluster.md b/docs/server/source/clusters-feds/set-up-a-cluster.md deleted file mode 100644 index 4b02bd9f..00000000 --- a/docs/server/source/clusters-feds/set-up-a-cluster.md +++ /dev/null @@ -1,28 +0,0 @@ -# Set Up a Cluster - -This section is about how to set up a BigchainDB cluster where each node is operated by a different operator. If you want to set up and run a testing cluster on AWS (where all nodes are operated by you), then see [the section about that](aws-testing-cluster.html). - - -## Initial Questions - -There are many questions that must be answered before setting up a BigchainDB cluster. For example: - -* Do you have a governance process for making consortium-level decisions, such as how to admit new members? -* What will you store in creation transactions (data payload)? Is there a data schema? -* Will you use transfer transactions? Will they include a non-empty data payload? -* Who will be allowed to submit transactions? Who will be allowed to read or query transactions? How will you enforce the access rules? - - -## Set Up the Initial Cluster - -The consortium must decide some things before setting up the initial cluster (initial set of BigchainDB nodes): - -1. Who will operate each node in the initial cluster? -2. What will the replication factor be? (It should be 3 or more.) -3. Who will deploy the first node, second node, etc.? - -Once those things have been decided, the cluster deployment process can begin. The process for deploying a production node is outlined in [the section on production nodes](../production-nodes/index.html). - -Every time a new BigchainDB node is added, every other node must update their [BigchainDB keyring](../server-reference/configuration.html#keyring) (one of the BigchainDB configuration settings): they must add the public key of the new node. - -To secure communications between BigchainDB nodes, each BigchainDB node can use a firewall or similar, and doing that will require additional coordination. diff --git a/docs/server/source/clusters.md b/docs/server/source/clusters.md new file mode 100644 index 00000000..52eb74de --- /dev/null +++ b/docs/server/source/clusters.md @@ -0,0 +1,50 @@ +# Clusters + +A **BigchainDB Cluster** is a set of connected **BigchainDB Nodes**, managed by a **BigchainDB Consortium** (i.e. an organization). Those terms are defined in the [BigchainDB Terminology page](https://docs.bigchaindb.com/en/latest/terminology.html). + + +## Consortium Structure & Governance + +The consortium might be a company, a foundation, a cooperative, or [some other form of organization](https://en.wikipedia.org/wiki/Organizational_structure). +It must make many decisions, e.g. How will new members be added? Who can read the stored data? What kind of data will be stored? +A governance process is required to make those decisions, and therefore one of the first steps for any new consortium is to specify its governance process (if one doesn't already exist). +This documentation doesn't explain how to create a consortium, nor does it outline the possible governance processes. + +It's worth noting that the decentralization of a BigchainDB cluster depends, +to some extent, on the decentralization of the associated consortium. See the pages about [decentralization](https://docs.bigchaindb.com/en/latest/decentralized.html) and [node diversity](https://docs.bigchaindb.com/en/latest/diversity.html). + + +## Relevant Technical Documentation + +There are some pages and sections that will be of particular interest to anyone building or managing a BigchainDB cluster. In particular: + +* [the page about how to set up and run a cluster node](production-nodes/setup-run-node.html), +* [our production deployment template](production-deployment-template/index.html), and +* [our old RethinkDB-based AWS deployment template](appendices/aws-testing-cluster.html). + + +## Cluster DNS Records and SSL Certificates + +We now describe how *we* set up the external (public-facing) DNS records for a BigchainDB cluster. Your consortium may opt to do it differently. +There were several goals: + +* Allow external users/clients to connect directly to any BigchainDB node in the cluster (over the internet), if they want. +* Each BigchainDB node operator should get an SSL certificate for their BigchainDB node, so that their BigchainDB node can serve the [BigchainDB HTTP API](http-client-server-api.html) via HTTPS. (The same certificate might also be used to serve the [WebSocket API](websocket-event-stream-api.html).) +* There should be no sharing of SSL certificates among BigchainDB node operators. +* Optional: Allow clients to connect to a "random" BigchainDB node in the cluster at one particular domain (or subdomain). + + +### Node Operator Responsibilities + +1. Register a domain (or use one that you already have) for your BigchainDB node. You can use a subdomain if you like. For example, you might opt to use `abc-org73.net`, `api.dynabob8.io` or `figmentdb3.ninja`. +2. Get an SSL certificate for your domain or subdomain, and properly install it in your node (e.g. in your NGINX instance). +3. Create a DNS A Record mapping your domain or subdomain to the public IP address of your node (i.e. the one that serves the BigchainDB HTTP API). + + +### Consortium Responsibilities + +Optional: The consortium managing the BigchainDB cluster could register a domain name and set up CNAME records mapping that domain name (or one of its subdomains) to each of the nodes in the cluster. For example, if the consortium registered `bdbcluster.io`, they could set up CNAME records like the following: + +* CNAME record mapping `api.bdbcluster.io` to `abc-org73.net` +* CNAME record mapping `api.bdbcluster.io` to `api.dynabob8.io` +* CNAME record mapping `api.bdbcluster.io` to `figmentdb3.ninja` diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index 312c6765..eefa81bb 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -1,21 +1,20 @@ -# The Digital Asset Model +# The Asset Model -To avoid redundant data in transactions, the digital asset model is different for `CREATE` and `TRANSFER` transactions. +To avoid redundant data in transactions, the asset model is different for `CREATE` and `TRANSFER` transactions. -A digital asset's properties are defined in a `CREATE` transaction with the following model: +In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair. The key must be `"data"` and the value can be any valid JSON document, or `null`. For example: ```json { - "data": "" + "data": { + "desc": "Gold-inlay bookmark owned by Xavier Bellomat Dickens III", + "xbd_collection_id": 1857 + } } ``` -For `TRANSFER` transactions we only keep the asset ID: +In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pair. They key must be `"id"` and the value must contain a transaction ID (i.e. a SHA3-256 hash: the ID of the `CREATE` transaction which created the asset, which also serves as the asset ID). For example: ```json { - "id": "" + "id": "38100137cea87fb9bd751e2372abb2c73e7d5bcf39d940a5516a324d9c7fb88d" } ``` - - -- `id`: The ID of the `CREATE` transaction that created the asset. -- `data`: A user supplied JSON document with custom information about the asset. Defaults to null. diff --git a/docs/server/source/data-models/index.rst b/docs/server/source/data-models/index.rst index 1e8b81c3..ccb36f3c 100644 --- a/docs/server/source/data-models/index.rst +++ b/docs/server/source/data-models/index.rst @@ -3,7 +3,7 @@ Data Models BigchainDB stores all data in the underlying database as JSON documents (conceptually, at least). There are three main kinds: -1. Transactions, which contain digital assets, inputs, outputs, and other things +1. Transactions, which contain assets, inputs, outputs, and other things 2. Blocks 3. Votes diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index 4309a4c8..da190477 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -81,30 +81,20 @@ to spend the asset. For example: { "condition": { "details": { - "bitmask": 41, - "subfulfillments": [ + "type": "threshold-sha-256", + "threshold": 2, + "subconditions": [ { - "bitmask": 32, "public_key": "", - "signature": null, - "type": "fulfillment", - "type_id": 4, - "weight": 1 + "type": "ed25519-sha-256", }, { - "bitmask": 32, "public_key": "", - "signature": null, - "type": "fulfillment", - "type_id": 4, - "weight": 1 + "type": "ed25519-sha-256", } ], - "threshold": 2, - "type": "fulfillment", - "type_id": 2 }, - "uri": "cc:2:29:ytNK3X6-bZsbF-nCGDTuopUIMi1HCyCkyPewm6oLI3o:206"}, + "uri": "ni:///sha-256;PNYwdxaRaNw60N6LDFzOWO97b8tJeragczakL8PrAPc?fpt=ed25519-sha-256&cost=131072"}, "public_keys": [ "", "" @@ -112,11 +102,10 @@ to spend the asset. For example: } -- ``subfulfillments``: a list of fulfillments - - ``weight``: integer weight for each subfulfillment's contribution to the threshold -- ``threshold``: threshold to reach for the subfulfillments to reach a valid fulfillment +- ``subconditions``: a list of condition specs +- ``threshold``: threshold to reach for the subconditions to reach a valid fulfillment -The ``weight``s and ``threshold`` could be adjusted. For example, if the ``threshold`` was changed to 1 above, then only one of the new owners would have to provide a signature to spend the asset. +The ``threshold`` can be adjusted. For example, if the ``threshold`` was changed to 1 above, then only one of the new owners would have to provide a signature to spend the asset. If it is desired to give a different weight to a subcondition, it should be specified multiple times. Inputs ------ @@ -132,8 +121,8 @@ If there is only one *current owner*, the fulfillment will be a simple signature "owners_before": [""], "fulfillment": "cf:4:RxFzIE679tFBk8zwEgizhmTuciAylvTUwy6EL6ehddHFJOhK5F4IjwQ1xLu2oQK9iyRCZJdfWAefZVjTt3DeG5j2exqxpGliOPYseNkRAWEakqJ_UrCwgnj92dnFRAEE", "fulfills": { - "output": 0, - "txid": "11b3e7d893cc5fdfcf1a1706809c7def290a3b10b0bef6525d10b024649c42d3" + "output_index": 0, + "transaction_id": "11b3e7d893cc5fdfcf1a1706809c7def290a3b10b0bef6525d10b024649c42d3" } } @@ -151,8 +140,8 @@ If there are multiple *current owners*, the fulfillment will be a little differe "owners_before": ["",""], "fulfillment": "cf:2:AQIBAgEBYwAEYEv6O5HjHGl7OWo2Tu5mWcWQcL_OGrFuUjyej-dK3LM99TbZsRd8c9luQhU30xCH5AdNaupxg-pLHuk8DoSaDA1MHQGXUZ80a_cV-4UaaaCpdey8K0CEcJxre0X96hTHCwABAWMABGBnsuHExhuSj5Mdm-q0KoPgX4nAt0s00k1WTMCzuUpQIp6aStLoTSMlsvS4fmDtOSv9gubekKLuHTMAk-LQFSKF1JdzwaVWAA2UOv0v_OS2gY3A-r0kRq8HtzjYdcmVswUA", "fulfills": { - "output": 0, - "txid": "e4805f1bfc999d6409b38e3a4c3b2fafad7c1280eb0d441da7083e945dd89eb8" + "output_index": 0, + "transaction_id": "e4805f1bfc999d6409b38e3a4c3b2fafad7c1280eb0d441da7083e945dd89eb8" } } @@ -160,5 +149,5 @@ If there are multiple *current owners*, the fulfillment will be a little differe - ``owners_before``: A list of public keys of the owners before the transaction; in this case it has two owners, hence two public keys. - ``fulfillment``: A crypto-conditions URI that encodes the cryptographic fulfillments like signatures and others;'cf' indicates this is a fulfillment, '2' indicates the condition type is THRESHOLD-SHA-256 (while '4' in `One Current Owner`_ indicates its condition type is ED25519). - ``fulfills``: Pointer to an output from a previous transaction that is being spent - - ``output``: The index of the output in a previous transaction - - ``txid``: ID of the transaction + - ``output_index``: The index of the output in a previous transaction + - ``transaction_id``: ID of the transaction diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index cc548aa9..38e523bd 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -1,17 +1,3 @@ -.. raw:: html - - - -===================== The Transaction Model ===================== @@ -20,33 +6,57 @@ A transaction has the following structure: .. code-block:: json { - "id": "", - "version": "", - "inputs": [""], - "outputs": [""], - "operation": "", - "asset": "", - "metadata": "" + "id": "", + "version": "", + "inputs": [""], + "outputs": [""], + "operation": "", + "asset": {""}, + "metadata": {""} } -Here's some explanation of the contents of a :ref:`transaction `: +Here's some explanation of the contents: -- id: The :ref:`id ` of the transaction, and also the database primary key. -- version: :ref:`Version ` number of the transaction model, so that software can support different transaction models. -- **inputs**: List of inputs. Each :ref:`input ` contains a pointer to an unspent output - and a *crypto fulfillment* that satisfies the conditions of that output. A *fulfillment* - is usually a signature proving the ownership of the asset. - See :doc:`./inputs-outputs`. +- **id**: The ID of the transaction and also the hash of the transaction (loosely speaking). See below for an explanation of how it's computed. It's also the database primary key. -- **outputs**: List of outputs. Each :ref:`output ` contains *crypto-conditions* that need to be fulfilled by a transfer transaction in order to transfer ownership to new owners. - See :doc:`./inputs-outputs`. +- **version**: The version-number of :ref:`the transaction schema `. As of BigchainDB Server 1.0.0, the only allowed value is ``"1.0"``. -- **operation**: String representation of the :ref:`operation ` being performed (currently either "CREATE", "TRANSFER" or "GENESIS"). It determines how the transaction should be validated. +- **inputs**: List of inputs. + Each input spends/transfers a previous output by satisfying/fulfilling + the crypto-conditions on that output. + A CREATE transaction should have exactly one input. + A TRANSFER transaction should have at least one input (i.e. ≥1). + For more details, see the subsection about :ref:`inputs `. -- **asset**: Definition of the digital :ref:`asset `. See next section. +- **outputs**: List of outputs. + Each output indicates the crypto-conditions which must be satisfied + by anyone wishing to spend/transfer that output. + It also indicates the number of shares of the asset tied to that output. + For more details, see the subsection about :ref:`outputs `. -- **metadata**: User-provided transaction :ref:`metadata `: Can be any JSON document, or `NULL`. +- **operation**: A string indicating what kind of transaction this is, + and how it should be validated. + It can only be ``"CREATE"``, ``"TRANSFER"`` or ``"GENESIS"`` + (but there should only be one transaction whose operation is ``"GENESIS"``: + the one in the GENESIS block). -Later, when we get to the models for the block and the vote, we'll see that both include a signature (from the node which created it). You may wonder why transactions don't have signatures... The answer is that they do! They're just hidden inside the ``fulfillment`` string of each input. A creation transaction is signed by whoever created it. A transfer transaction is signed by whoever currently controls or owns it. +- **asset**: A JSON document for the asset associated with the transaction. + (A transaction can only be associated with one asset.) + See :ref:`the page about the asset model `. -What gets signed? For each input in the transaction, the "fullfillment message" that gets signed includes the JSON serialized body of the transaction, minus any fulfillment strings. The computed signature goes into creating the ``fulfillment`` string of the input. +- **metadata**: User-provided transaction metadata. + It can be any valid JSON document, or ``null``. + +**How the transaction ID is computed.** +1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, +2) In each of the inputs, replace the value of each ``fulfillment`` with ``null``, +3) :ref:`Serialize ` that dictionary, +4) The transaction ID is just :ref:`the SHA3-256 hash ` of the serialized dictionary. + +**About signing the transaction.** +Later, when we get to the models for the block and the vote, we'll see that both include a signature (from the node which created it). You may wonder why transactions don't have signatures… The answer is that they do! They're just hidden inside the ``fulfillment`` string of each input. What gets signed (as of version 1.0.0) is everything inside the transaction, including the ``id``, but the value of each ``fulfillment`` is replaced with ``null``. + +There are example BigchainDB transactions in +:ref:`the HTTP API documentation ` +and +`the Python Driver documentation `_. diff --git a/docs/server/source/data-models/vote-model.md b/docs/server/source/data-models/vote-model.md index 25d5029c..daa66a94 100644 --- a/docs/server/source/data-models/vote-model.md +++ b/docs/server/source/data-models/vote-model.md @@ -4,17 +4,24 @@ A vote has the following structure: ```json { - "id": "", - "node_pubkey": "", + "node_pubkey": "", "vote": { - "voting_for_block": "", - "previous_block": "", - "is_block_valid": "", - "invalid_reason": "", + "previous_block": "", + "is_block_valid": "", + "invalid_reason": null, "timestamp": "" }, - "signature": "" + "signature": "" } ``` -Note: The `invalid_reason` was not being used and may be dropped in a future version of BigchainDB. See [Issue #217](https://github.com/bigchaindb/bigchaindb/issues/217) on GitHub. +**Notes** + +* Votes have no ID (or `"id"`), as far as users are concerned. (The backend database uses one internally, but it's of no concern to users and it's never reported to them via BigchainDB APIs.) + +* At the time of writing, the value of `"invalid_reason"` was always `null`. In other words, it wasn't being used. It may be used or dropped in a future version of BigchainDB. See [Issue #217](https://github.com/bigchaindb/bigchaindb/issues/217) on GitHub. + +* For more information about the vote `"timestamp"`, see [the page about timestamps in BigchainDB](https://docs.bigchaindb.com/en/latest/timestamps.html). + +* For more information about how the `"signature"` is calculated, see [the page about cryptography in BigchainDB](../appendices/cryptography.html). diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index ef749d55..f6e5c6ae 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -4,7 +4,8 @@ Drivers & Clients Libraries and Tools Maintained by the BigchainDB Team ----------------------------------------------------- -* `The Python Driver `_ +* `Python Driver `_ +* `JavaScript / Node.js Driver `_ * `The Transaction CLI `_ is a command-line interface for building BigchainDB transactions. You may be able to call it from inside the language of @@ -20,7 +21,6 @@ Community-Driven Libraries and Tools Some of these projects are a work in progress, but may still be useful. -* `JavaScript / Node.js driver `_ * `Haskell transaction builder `_ * `Go driver `_ * `Java driver `_ diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 9fd7aee5..618903a8 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -42,19 +42,19 @@ that allows you to discover the BigchainDB API endpoints: Transactions ------------------- -.. http:get:: /api/v1/transactions/{tx_id} +.. http:get:: /api/v1/transactions/{transaction_id} - Get the transaction with the ID ``tx_id``. + Get the transaction with the ID ``transaction_id``. - This endpoint returns a transaction if it was included in a ``VALID`` block, - if it is still waiting to be processed (``BACKLOG``) or is still in an - undecided block (``UNDECIDED``). All instances of a transaction in invalid - blocks are ignored and treated as if they don't exist. If a request is made - for a transaction and instances of that transaction are found only in - invalid blocks, then the response will be ``404 Not Found``. + This endpoint returns a transaction if it was included in a ``VALID`` block. + All instances of a transaction in invalid/undecided blocks or the backlog + are ignored and treated as if they don't exist. If a request is made for a + transaction and instances of that transaction are found only in + invalid/undecided blocks or the backlog, then the response will be ``404 Not + Found``. - :param tx_id: transaction ID - :type tx_id: hex string + :param transaction_id: transaction ID + :type transaction_id: hex string **Example request**: @@ -147,7 +147,16 @@ Transactions .. literalinclude:: http-samples/post-tx-response.http :language: http + .. note:: + If the server is returning a ``202`` HTTP status code, then the + transaction has been accepted for processing. To check the status of the + transaction, poll the link to the + :ref:`status monitor ` + provided in the ``Location`` header or listen to server's + :ref:`WebSocket Event Stream API `. + :resheader Content-Type: ``application/json`` + :resheader Location: Relative link to a status monitor for the submitted transaction. :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. @@ -157,21 +166,29 @@ Transaction Outputs ------------------- The ``/api/v1/outputs`` endpoint returns transactions outputs filtered by a -given public key, and optionally filtered to only include outputs that have -not already been spent. +given public key, and optionally filtered to only include either spent or +unspent outputs. -.. http:get:: /api/v1/outputs?public_key={public_key} +.. http:get:: /api/v1/outputs - Get transaction outputs by public key. The `public_key` parameter must be + Get transaction outputs by public key. The ``public_key`` parameter must be a base58 encoded ed25519 public key associated with transaction output ownership. - Returns a list of links to transaction outputs. + Returns a list of transaction outputs. - :param public_key: Base58 encoded public key associated with output ownership. This parameter is mandatory and without it the endpoint will return a ``400`` response code. - :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend. Defaults to "false". + :param public_key: Base58 encoded public key associated with output + ownership. This parameter is mandatory and without it + the endpoint will return a ``400`` response code. + :param spent: Boolean value ("true" or "false") indicating if the result set + should include only spent or only unspent outputs. If not + specified the result includes all the outputs (both spent + and unspent) associated with the ``public_key``. +.. http:get:: /api/v1/outputs?public_key={public_key} + + Return all outputs, both spent and unspent, for the ``public_key``. **Example request**: @@ -188,8 +205,70 @@ not already been spent. Content-Type: application/json [ - "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/0", - "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/1" + { + "output_index": 0, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + }, + { + "output_index": 1, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } + ] + + :statuscode 200: A list of outputs were found and returned in the body of the response. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. + +.. http:get:: /api/v1/outputs?public_key={public_key}&spent=true + + Return all **spent** outputs for ``public_key``. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/outputs?public_key=1AAAbbb...ccc&spent=true HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "output_index": 0, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } + ] + + :statuscode 200: A list of outputs were found and returned in the body of the response. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. + +.. http:get:: /api/v1/outputs?public_key={public_key}&spent=false + + Return all **unspent** outputs for ``public_key``. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/outputs?public_key=1AAAbbb...ccc&spent=false HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "output_index": 1, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } ] :statuscode 200: A list of outputs were found and returned in the body of the response. @@ -203,21 +282,19 @@ Statuses Get the status of an asynchronously written transaction or block by their id. - A link to the resource is also provided in the returned payload under - ``_links``. - - :query string tx_id: transaction ID + :query string transaction_id: transaction ID :query string block_id: block ID .. note:: - Exactly one of the ``tx_id`` or ``block_id`` query parameters must be + Exactly one of the ``transaction_id`` or ``block_id`` query parameters must be used together with this endpoint (see below for getting `transaction statuses <#get--statuses?tx_id=tx_id>`_ and `block statuses <#get--statuses?block_id=block_id>`_). +.. _get_status_of_transaction: -.. http:get:: /api/v1/statuses?tx_id={tx_id} +.. http:get:: /api/v1/statuses?transaction_id={transaction_id} Get the status of a transaction. @@ -236,7 +313,6 @@ Statuses :language: http :resheader Content-Type: ``application/json`` - :resheader Location: Once the transaction has been persisted, this header will link to the actual resource. :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. @@ -255,16 +331,10 @@ Statuses **Example response**: - .. literalinclude:: http-samples/get-statuses-block-invalid-response.http - :language: http - - **Example response**: - .. literalinclude:: http-samples/get-statuses-block-valid-response.http :language: http :resheader Content-Type: ``application/json`` - :resheader Location: Once the block has been persisted, this header will link to the actual resource. :statuscode 200: A block with that ID was found. :statuscode 404: A block with that ID was not found. @@ -288,8 +358,8 @@ Assets .. http:get:: /api/v1/assets?search={text_search} - Return all assets that match a given text search. The asset is returned - with the ``id`` of the transaction that created the asset. + Return all assets that match a given text search. The ``id`` of the asset + is the same ``id`` of the transaction that created the asset. If no assets match the text search it returns an empty list. @@ -388,12 +458,12 @@ Advanced Usage The following endpoints are more advanced and meant for debugging and transparency purposes. More precisely, the `blocks endpoint <#blocks>`_ allows you to retrieve a block by ``block_id`` as well the list of blocks that -a certain transaction with ``tx_id`` occured in (a transaction can occur in multiple ``invalid`` blocks until it +a certain transaction with ``transaction_id`` occured in (a transaction can occur in multiple ``invalid`` blocks until it either gets rejected or validated by the system). This endpoint gives the ability to drill down on the lifecycle of a transaction The `votes endpoint <#votes>`_ contains all the voting information for a specific block. So after retrieving the -``block_id`` for a given ``tx_id``, one can now simply inspect the votes that happened at a specific time on that block. +``block_id`` for a given ``transaction_id``, one can now simply inspect the votes that happened at a specific time on that block. Blocks @@ -429,8 +499,8 @@ Blocks .. http:get:: /api/v1/blocks The unfiltered ``/blocks`` endpoint without any query parameters returns a `400` status code. - The list endpoint should be filtered with a ``tx_id`` query parameter, - see the ``/blocks?tx_id={tx_id}&status={UNDECIDED|VALID|INVALID}`` + The list endpoint should be filtered with a ``transaction_id`` query parameter, + see the ``/blocks?transaction_id={transaction_id}&status={UNDECIDED|VALID|INVALID}`` `endpoint <#get--blocks?tx_id=tx_id&status=UNDECIDED|VALID|INVALID>`_. @@ -449,9 +519,9 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. -.. http:get:: /api/v1/blocks?tx_id={tx_id}&status={UNDECIDED|VALID|INVALID} +.. http:get:: /api/v1/blocks?transaction_id={transaction_id}&status={UNDECIDED|VALID|INVALID} - Retrieve a list of ``block_id`` with their corresponding status that contain a transaction with the ID ``tx_id``. + Retrieve a list of ``block_id`` with their corresponding status that contain a transaction with the ID ``transaction_id``. Any blocks, be they ``UNDECIDED``, ``VALID`` or ``INVALID`` will be returned if no status filter is provided. @@ -460,7 +530,7 @@ Blocks In case no block was found, an empty list and an HTTP status code ``200 OK`` is returned, as the request was still successful. - :query string tx_id: transaction ID *(required)* + :query string transaction_id: transaction ID *(required)* :query string status: Filter blocks by their status. One of ``VALID``, ``UNDECIDED`` or ``INVALID``. **Example request**: @@ -475,8 +545,8 @@ Blocks :resheader Content-Type: ``application/json`` - :statuscode 200: A list of blocks containing a transaction with ID ``tx_id`` was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``tx_id``. + :statuscode 200: A list of blocks containing a transaction with ID ``transaction_id`` was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``transaction_id``. Votes diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index 0dee9174..f2f86a61 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -8,8 +8,8 @@ BigchainDB Server Documentation introduction quickstart production-nodes/index - clusters-feds/index - cloud-deployment-templates/index + clusters + production-deployment-template/index dev-and-test/index server-reference/index http-client-server-api diff --git a/docs/server/source/introduction.md b/docs/server/source/introduction.md index 02cf5ecf..58d7de16 100644 --- a/docs/server/source/introduction.md +++ b/docs/server/source/introduction.md @@ -8,7 +8,7 @@ Note that there are a few kinds of nodes: - A **dev/test node** is a node created by a developer working on BigchainDB Server, e.g. for testing new or changed code. A dev/test node is typically run on the developer's local machine. -- A **bare-bones node** is a node deployed in the cloud, either as part of a testing cluster or as a starting point before upgrading the node to be production-ready. Our cloud deployment templates deploy a bare-bones node, as do our scripts for deploying a testing cluster on AWS. +- A **bare-bones node** is a node deployed in the cloud, either as part of a testing cluster or as a starting point before upgrading the node to be production-ready. - A **production node** is a node that is part of a consortium's BigchainDB cluster. A production node has the most components and requirements. @@ -16,10 +16,14 @@ Note that there are a few kinds of nodes: ## Setup Instructions for Various Cases * [Set up a local stand-alone BigchainDB node for learning and experimenting: Quickstart](quickstart.html) -* [Set up and run a bare-bones node in the cloud](cloud-deployment-templates/index.html) * [Set up and run a local dev/test node for developing and testing BigchainDB Server](dev-and-test/setup-run-node.html) -* [Deploy a testing cluster on AWS](clusters-feds/aws-testing-cluster.html) -* [Set up and run a cluster (including production nodes)](clusters-feds/set-up-a-cluster.html) +* [Set up and run a BigchainDB cluster](clusters.html) + +There are some old RethinkDB-based deployment instructions as well: + +* [Deploy a bare-bones RethinkDB-based node on Azure](appendices/azure-quickstart-template.html) +* [Deploy a bare-bones RethinkDB-based node on any Ubuntu machine with Ansible](appendices/template-ansible.html) +* [Deploy a RethinkDB-based testing cluster on AWS](appendices/aws-testing-cluster.html) Instructions for setting up a client will be provided once there's a public test net. diff --git a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst b/docs/server/source/production-deployment-template/add-node-on-kubernetes.rst similarity index 92% rename from docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst rename to docs/server/source/production-deployment-template/add-node-on-kubernetes.rst index 7dcf1104..fd0611e6 100644 --- a/docs/server/source/cloud-deployment-templates/add-node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/add-node-on-kubernetes.rst @@ -71,10 +71,10 @@ Step 2: Prepare the New Kubernetes Cluster Follow the steps in the sections to set up Storage Classes and Persistent Volume Claims, and to run MongoDB in the new cluster: -1. :ref:`Add Storage Classes ` -2. :ref:`Add Persistent Volume Claims ` -3. :ref:`Create the Config Map ` -4. :ref:`Run MongoDB instance ` +1. :ref:`Add Storage Classes `. +2. :ref:`Add Persistent Volume Claims `. +3. :ref:`Create the Config Map `. +4. :ref:`Run MongoDB instance `. Step 3: Add the New MongoDB Instance to the Existing Replica Set @@ -166,13 +166,13 @@ show-config`` command to check that the keyring is updated. Step 7: Run NGINX as a Deployment --------------------------------- -Please refer :ref:`this ` to +Please see :ref:`this page ` to set up NGINX in your new node. Step 8: Test Your New BigchainDB Node ------------------------------------- -Please refer to the testing steps :ref:`here ` to verify that your new BigchainDB node is working as expected. diff --git a/docs/server/source/cloud-deployment-templates/ca-installation.rst b/docs/server/source/production-deployment-template/ca-installation.rst similarity index 83% rename from docs/server/source/cloud-deployment-templates/ca-installation.rst rename to docs/server/source/production-deployment-template/ca-installation.rst index 9ea38477..146bd461 100644 --- a/docs/server/source/cloud-deployment-templates/ca-installation.rst +++ b/docs/server/source/production-deployment-template/ca-installation.rst @@ -33,13 +33,17 @@ by going to the ``bdb-cluster-ca/easy-rsa-3.0.1/easyrsa3`` directory and using: ./easyrsa build-ca - -You will be asked to enter a PEM pass phrase for encrypting the ``ca.key`` file. +You will also be asked to enter a PEM pass phrase (for encrypting the ``ca.key`` file). Make sure to securely store that PEM pass phrase. If you lose it, you won't be able to add or remove entities from your PKI infrastructure in the future. -It will ask several other questions. -You can accept all the defaults [in brackets] by pressing Enter. +You will be prompted to enter the Distinguished Name (DN) information for this CA. +For each field, you can accept the default value [in brackets] by pressing Enter. + +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value ``ROOT-CA``. + While ``Easy-RSA CA`` *is* a valid and acceptable Common Name, you should probably enter a name based on the name of the managing organization, e.g. ``Omega Ledger CA``. @@ -51,7 +55,7 @@ by using the subcommand ``./easyrsa help`` Step 3: Create an Intermediate CA --------------------------------- -TODO(Krish) +TODO Step 4: Generate a Certificate Revocation List ---------------------------------------------- @@ -62,9 +66,9 @@ You can generate a Certificate Revocation List (CRL) using: ./easyrsa gen-crl -You will need to run this command every time you revoke a certificate and the -generated ``crl.pem`` needs to be uploaded to your infrastructure to prevent -the revoked certificate from being used again. +You will need to run this command every time you revoke a certificate. +The generated ``crl.pem`` needs to be uploaded to your infrastructure to +prevent the revoked certificate from being used again. Step 5: Secure the CA diff --git a/docs/server/source/cloud-deployment-templates/client-tls-certificate.rst b/docs/server/source/production-deployment-template/client-tls-certificate.rst similarity index 56% rename from docs/server/source/cloud-deployment-templates/client-tls-certificate.rst rename to docs/server/source/production-deployment-template/client-tls-certificate.rst index 60a754a0..5a729836 100644 --- a/docs/server/source/cloud-deployment-templates/client-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/client-tls-certificate.rst @@ -1,9 +1,8 @@ How to Generate a Client Certificate for MongoDB ================================================ -This page enumerates the steps *we* use -to generate a client certificate -to be used by clients who want to connect to a TLS-secured MongoDB cluster. +This page enumerates the steps *we* use to generate a client certificate to be +used by clients who want to connect to a TLS-secured MongoDB cluster. We use Easy-RSA. @@ -25,7 +24,7 @@ Step 2: Create the Client Private Key and CSR --------------------------------------------- You can create the client private key and certificate signing request (CSR) -by going into the directory ``client-cert/easy-rsa-3.0.1/easyrsa`` +by going into the directory ``client-cert/easy-rsa-3.0.1/easyrsa3`` and using: .. code:: bash @@ -34,26 +33,37 @@ and using: ./easyrsa gen-req bdb-instance-0 nopass +You should change the Common Name (e.g. ``bdb-instance-0``) +to a value that reflects what the +client certificate is being used for, e.g. ``mdb-mon-instance-3`` or ``mdb-bak-instance-4``. (The final integer is specific to your BigchainDB node in the BigchainDB cluster.) -You should change ``bdb-instance-0`` to a value based on the client -the certificate is for. +You will be prompted to enter the Distinguished Name (DN) information for this certificate. For each field, you can accept the default value [in brackets] by pressing Enter. -Tip: You can get help with the ``easyrsa`` command (and its subcommands) -by using the subcommand ``./easyrsa help`` +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value + ``BigchainDB-Instance``, ``MongoDB-Mon-Instance`` or ``MongoDB-Backup-Instance`` + as appropriate. + +Aside: The ``nopass`` option means "do not encrypt the private key (default is encrypted)". You can get help with the ``easyrsa`` command (and its subcommands) +by using the subcommand ``./easyrsa help``. Step 3: Get the Client Certificate Signed ----------------------------------------- -The CSR file (created in the last step) -should be located in ``pki/reqs/bdb-instance-0.req``. +The CSR file created in the previous step +should be located in ``pki/reqs/bdb-instance-0.req`` +(or whatever Common Name you used in the ``gen-req`` command above). You need to send it to the organization managing the cluster so that they can use their CA to sign the request. (The managing organization should already have a self-signed CA.) If you are the admin of the managing organization's self-signed CA, -then you can import the CSR and use Easy-RSA to sign it. For example: +then you can import the CSR and use Easy-RSA to sign it. +Go to your ``bdb-cluster-ca/easy-rsa-3.0.1/easyrsa3/`` +directory and do something like: .. code:: bash diff --git a/docs/server/source/production-deployment-template/cloud-manager.rst b/docs/server/source/production-deployment-template/cloud-manager.rst new file mode 100644 index 00000000..0a1030a3 --- /dev/null +++ b/docs/server/source/production-deployment-template/cloud-manager.rst @@ -0,0 +1,96 @@ +Configure MongoDB Cloud Manager for Monitoring and Backup +========================================================= + +This document details the steps required to configure MongoDB Cloud Manager to +enable monitoring and backup of data in a MongoDB Replica Set. + + +Configure MongoDB Cloud Manager for Monitoring +---------------------------------------------- + + * Once the Monitoring Agent is up and running, open + `MongoDB Cloud Manager `_. + + * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud + Manager. + + * Select the group from the dropdown box on the page. + + * Go to Settings, Group Settings and add a ``Preferred Hostnames`` entry as + a regexp based on the ``mdb-instance-name`` of the nodes in your cluster. + It may take up to 5 mins till this setting takes effect. + You may refresh the browser window and verify whether the changes have + been saved or not. + + For example, for the nodes in a cluster that are named ``mdb-instance-0``, + ``mdb-instance-1`` and so on, a regex like ``^mdb-instance-[0-9]{1,2}$`` + is recommended. + + * Next, click the ``Deployment`` tab, and then the ``Manage Existing`` + button. + + * On the ``Import your deployment for monitoring`` page, enter the hostname + to be the same as the one set for ``mdb-instance-name`` in the global + ConfigMap for a node. + For example, if the ``mdb-instance-name`` is set to ``mdb-instance-0``, + enter ``mdb-instance-0`` as the value in this field. + + * Enter the port number as ``27017``, with no authentication. + + * If you have authentication enabled, select the option to enable + authentication and specify the authentication mechanism as per your + deployment. The default BigchainDB production deployment currently + supports ``X.509 Client Certificate`` as the authentication mechanism. + + * If you have TLS enabled, select the option to enable TLS/SSL for MongoDB + connections, and click ``Continue``. This should already be selected for + you in case you selected ``X.509 Client Certificate`` above. + + * Wait a minute or two for the deployment to be found and then + click the ``Continue`` button again. + + * Verify that you see your process on the Cloud Manager UI. + It should look something like this: + + .. image:: /_static/mongodb_cloud_manager_1.png + + * Click ``Continue``. + + * Verify on the UI that data is being sent by the monitoring agent to the + Cloud Manager. It may take upto 5 minutes for data to appear on the UI. + + +Configure MongoDB Cloud Manager for Backup +------------------------------------------ + + * Once the Backup Agent is up and running, open + `MongoDB Cloud Manager `_. + + * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud + Manager. + + * Select the group from the dropdown box on the page. + + * Click ``Backup`` tab. + + * Hover over the ``Status`` column of your backup and click ``Start`` + to start the backup. + + * Select the replica set on the side pane. + + * If you have authentication enabled, select the authentication mechanism as + per your deployment. The default BigchainDB production deployment currently + supports ``X.509 Client Certificate`` as the authentication mechanism. + + * If you have TLS enabled, select the checkbox ``Replica set allows TLS/SSL + connections``. This should be selected by default in case you selected + ``X.509 Client Certificate`` as the auth mechanism above. + + * Choose the ``WiredTiger`` storage engine. + + * Verify the details of your MongoDB instance and click on ``Start``. + + * It may take up to 5 minutes for the backup process to start. + During this process, the UI will show the status of the backup process. + + * Verify that data is being backed up on the UI. diff --git a/docs/server/source/cloud-deployment-templates/easy-rsa.rst b/docs/server/source/production-deployment-template/easy-rsa.rst similarity index 67% rename from docs/server/source/cloud-deployment-templates/easy-rsa.rst rename to docs/server/source/production-deployment-template/easy-rsa.rst index 50a62cd5..ff268bf2 100644 --- a/docs/server/source/cloud-deployment-templates/easy-rsa.rst +++ b/docs/server/source/production-deployment-template/easy-rsa.rst @@ -48,10 +48,10 @@ by copying the existing ``vars.example`` file and then editing it. You should change the country, province, city, org and email -to the correct values for you. +to the correct values for your organisation. (Note: The country, province, city, org and email are part of the `Distinguished Name `_ (DN).) -The comments in the file explain what the variables mean. +The comments in the file explain what each of the variables mean. .. code:: bash @@ -60,25 +60,31 @@ The comments in the file explain what the variables mean. cp vars.example vars echo 'set_var EASYRSA_DN "org"' >> vars - echo 'set_var EASYRSA_REQ_OU "IT"' >> vars echo 'set_var EASYRSA_KEY_SIZE 4096' >> vars - + echo 'set_var EASYRSA_REQ_COUNTRY "DE"' >> vars echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> vars echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> vars echo 'set_var EASYRSA_REQ_ORG "BigchainDB GmbH"' >> vars + echo 'set_var EASYRSA_REQ_OU "IT"' >> vars echo 'set_var EASYRSA_REQ_EMAIL "dev@bigchaindb.com"' >> vars +Note: Later, when building a CA or generating a certificate signing request, you will be prompted to enter a value for the OU (or to accept the default). You should change the default OU from ``IT`` to one of the following, as appropriate: +``ROOT-CA``, +``MongoDB-Instance``, ``BigchainDB-Instance``, ``MongoDB-Mon-Instance`` or +``MongoDB-Backup-Instance``. +To understand why, see `the MongoDB Manual `_. +There are reminders to do this in the relevant docs. + Step 4: Maybe Edit x509-types/server ------------------------------------ .. warning:: - Only do this step if you are setting up a self-signed CA - or creating a server/member certificate. + Only do this step if you are setting up a self-signed CA. -Edit the file ``x509-types/server`` and change -``extendedKeyUsage = serverAuth`` to -``extendedKeyUsage = serverAuth,clientAuth``. -See `the MongoDB documentation about x.509 authentication `_ to understand why. + Edit the file ``x509-types/server`` and change + ``extendedKeyUsage = serverAuth`` to + ``extendedKeyUsage = serverAuth,clientAuth``. + See `the MongoDB documentation about x.509 authentication `_ to understand why. diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/production-deployment-template/index.rst similarity index 93% rename from docs/server/source/cloud-deployment-templates/index.rst rename to docs/server/source/production-deployment-template/index.rst index a94fab94..766c69b9 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/production-deployment-template/index.rst @@ -22,6 +22,7 @@ Feel free change things to suit your needs or preferences. node-on-kubernetes add-node-on-kubernetes upgrade-on-kubernetes - first-node log-analytics easy-rsa + cloud-manager + node-config-map-and-secrets diff --git a/docs/server/source/cloud-deployment-templates/log-analytics.rst b/docs/server/source/production-deployment-template/log-analytics.rst similarity index 83% rename from docs/server/source/cloud-deployment-templates/log-analytics.rst rename to docs/server/source/production-deployment-template/log-analytics.rst index 1f5d5596..5354b4e2 100644 --- a/docs/server/source/cloud-deployment-templates/log-analytics.rst +++ b/docs/server/source/production-deployment-template/log-analytics.rst @@ -193,30 +193,55 @@ simply run the following command: $ kubectl create -f oms-daemonset.yaml -Create an Email Alert ---------------------- +Search the OMS Logs +------------------- -Suppose you want to get an email whenever there's a logging message -with the CRITICAL or ERROR logging level from any container. -At the time of writing, it wasn't possible to create email alerts -using the Azure Portal (as far as we could tell), -but it *was* possible using the OMS Portal. -(There are instructions to get to the OMS Portal -in the section titled :ref:`Deploy the OMS Agents` above.) +OMS should now be getting, storing and indexing all the logs +from all the containers in your Kubernetes cluster. +You can search the OMS logs from the Azure Portal +or the OMS Portal, but at the time of writing, +there was more functionality in the OMS Portal +(e.g. the ability to create an Alert based on a search). + +There are instructions to get to the OMS Portal +in the section titled :ref:`Deploy the OMS Agents` above. Once you're in the OMS Portal, click on **Log Search** -and enter the query string: +and enter a query. +Here are some example queries: + +All logging messages containing the strings "critical" or "error" (not case-sensitive): ``Type=ContainerLog (critical OR error)`` -If you don't see any query results, -try experimenting with the query string and time range -to convince yourself that it's working. -For query syntax help, see the -`Log Analytics search reference `_. -If you want to exclude the "404 Not Found" errors, -use the query string -"Type=ContainerLog (critical OR error) NOT(404)". -Once you're satisfied with the query string, +.. note:: + + You can filter the results even more by clicking on things in the left sidebar. + For OMS Log Search syntax help, see the + `Log Analytics search reference `_. + +All logging messages containing the string "error" but not "404": + +``Type=ContainerLog error NOT(404)`` + +All logging messages containing the string "critical" but not "CriticalAddonsOnly": + +``Type=ContainerLog critical NOT(CriticalAddonsOnly)`` + +All logging messages from containers running the Docker image bigchaindb/nginx_3scale:1.3, containing the string "GET" but not the strings "Go-http-client" or "runscope" (where those exclusions filter out tests by Kubernetes and Runscope): + +``Type=ContainerLog Image="bigchaindb/nginx_3scale:1.3" GET NOT("Go-http-client") NOT(runscope)`` + +.. note:: + + We wrote a small Python 3 script to analyze the logs found by the above NGINX search. + It's in ``k8s/logging-and-monitoring/analyze.py``. The docsting at the top + of the script explains how to use it. + + +Create an Email Alert +--------------------- + +Once you're satisfied with an OMS Log Search query string, click the **🔔 Alert** icon in the top menu, fill in the form, and click **Save** when you're done. diff --git a/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst new file mode 100644 index 00000000..7bcbb28d --- /dev/null +++ b/docs/server/source/production-deployment-template/node-config-map-and-secrets.rst @@ -0,0 +1,161 @@ +How to Configure a BigchainDB Node +================================== + +This page outlines the steps to set a bunch of configuration settings +in your BigchainDB node. +They are pushed to the Kubernetes cluster in two files, +named ``config-map.yaml`` (a set of ConfigMaps) +and ``secret.yaml`` (a set of Secrets). +They are stored in the Kubernetes cluster's key-value store (etcd). + +Make sure you did all the things listed in the section titled +:ref:`Things Each Node Operator Must Do` +(including generation of all the SSL certificates needed +for MongoDB auth). + + +Edit config-map.yaml +-------------------- + +Make a copy of the file ``k8s/configuration/config-map.yaml`` +and edit the data values in the various ConfigMaps. +That file already contains many comments to help you +understand each data value, but we make some additional +remarks on some of the values below. + +Note: None of the data values in ``config-map.yaml`` need +to be base64-encoded. (This is unlike ``secret.yaml``, +where all data values must be base64-encoded. +This is true of all Kubernetes ConfigMaps and Secrets.) + + +vars.mdb-instance-name and Similar +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Your BigchainDB cluster organization should have a standard way +of naming instances, so the instances in your BigchainDB node +should conform to that standard (i.e. you can't just make up some names). +There are some things worth noting about the ``mdb-instance-name``: + +* MongoDB reads the local ``/etc/hosts`` file while bootstrapping a replica + set to resolve the hostname provided to the ``rs.initiate()`` command. + It needs to ensure that the replica set is being initialized in the same + instance where the MongoDB instance is running. +* We use the value in the ``mdb-instance-name`` field to achieve this. +* This field will be the DNS name of your MongoDB instance, and Kubernetes + maps this name to its internal DNS. +* This field will also be used by other MongoDB instances when forming a + MongoDB replica set. +* We use ``mdb-instance-0``, ``mdb-instance-1`` and so on in our + documentation. Your BigchainDB cluster may use a different naming convention. + +bdb-keyring.bdb-keyring +~~~~~~~~~~~~~~~~~~~~~~~ + +This lists the BigchainDB public keys +of all *other* nodes in your BigchainDB cluster +(not including the public key of your BigchainDB node). Cases: + +* If you're deploying the first node in the cluster, + the value should be ``""`` (an empty string). +* If you're deploying the second node in the cluster, + the value should be the BigchainDB public key of the first/original + node in the cluster. + For example, + ``"EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"`` +* If there are two or more other nodes already in the cluster, + the value should be a colon-separated list + of the BigchainDB public keys + of those other nodes. + For example, + ``"DPjpKbmbPYPKVAuf6VSkqGCf5jzrEh69Ldef6TrLwsEQ:EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"`` + + +Edit secret.yaml +---------------- + +Make a copy of the file ``k8s/configuration/secret.yaml`` +and edit the data values in the various Secrets. +That file includes many comments to explain the required values. +**In particular, note that all values must be base64-encoded.** +There are tips at the top of the file +explaining how to convert values into base64-encoded values. + +Your BigchainDB node might not need all the Secrets. +For example, if you plan to access the BigchainDB API over HTTP, you +don't need the ``https-certs`` Secret. +You can delete the Secrets you don't need, +or set their data values to ``""``. + +Note that ``ca.pem`` is just another name for ``ca.crt`` +(the certificate of your BigchainDB cluster's self-signed CA). + + +bdb-certs.bdb-user +~~~~~~~~~~~~~~~~~~ + +This is the user name that BigchainDB uses to authenticate itself to the +backend MongoDB database. + +We need to specify the user name *as seen in the certificate* issued to +the BigchainDB instance in order to authenticate correctly. Use +the following ``openssl`` command to extract the user name from the +certificate: + +.. code:: bash + + $ openssl x509 -in \ + -inform PEM -subject -nameopt RFC2253 + +You should see an output line that resembles: + +.. code:: bash + + subject= emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE + +The ``subject`` line states the complete user name we need to use for this +field (``bdb-certs.bdb-user``), i.e. + +.. code:: bash + + emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE + + +threescale-credentials.* +~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're not using 3scale, +you can delete the ``threescale-credentials`` Secret +or leave all the values blank (``""``). + +If you *are* using 3scale, you can get the value for ``frontend-api-dns-name`` +using something like ``echo "your.nodesubdomain.net" | base64 -w 0`` + +To get the values for ``secret-token``, ``service-id``, +``version-header`` and ``provider-key``, login to your 3scale admin, +then click **APIs** and click on **Integration** for the relevant API. +Scroll to the bottom of the page and click the small link +in the lower right corner, labelled **Download the NGINX Config files**. +You'll get a ``.zip`` file. +Unzip it, then open the ``.conf`` file and the ``.lua`` file. +You should be able to find all the values in those files. +You have to be careful because it will have values for *all* your APIs, +and some values vary from API to API. +The ``version-header`` is the timestamp in a line that looks like: + +.. code:: + + proxy_set_header X-3scale-Version "2017-06-28T14:57:34Z"; + + +Deploy Your config-map.yaml and secret.yaml +------------------------------------------- + +You can deploy your edited ``config-map.yaml`` and ``secret.yaml`` +files to your Kubernetes cluster using the commands: + +.. code:: bash + + $ kubectl apply -f config-map.yaml + + $ kubectl apply -f secret.yaml diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst new file mode 100644 index 00000000..3ee7d2c5 --- /dev/null +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -0,0 +1,818 @@ +Kubernetes Template: Deploy a Single BigchainDB Node +==================================================== + +This page describes how to deploy the first BigchainDB node +in a BigchainDB cluster, or a stand-alone BigchainDB node, +using `Kubernetes `_. +It assumes you already have a running Kubernetes cluster. + +If you want to add a new BigchainDB node to an existing BigchainDB cluster, +refer to :doc:`the page about that `. + +Below, we refer to many files by their directory and filename, +such as ``configuration/config-map.yaml``. Those files are files in the +`bigchaindb/bigchaindb repository on GitHub +`_ in the ``k8s/`` directory. +Make sure you're getting those files from the appropriate Git branch on +GitHub, i.e. the branch for the version of BigchainDB that your BigchainDB +cluster is using. + + +Step 1: Install and Configure kubectl +------------------------------------- + +kubectl is the Kubernetes CLI. +If you don't already have it installed, +then see the `Kubernetes docs to install it +`_. + +The default location of the kubectl configuration file is ``~/.kube/config``. +If you don't have that file, then you need to get it. + +**Azure.** If you deployed your Kubernetes cluster on Azure +using the Azure CLI 2.0 (as per :doc:`our template `), +then you can get the ``~/.kube/config`` file using: + +.. code:: bash + + $ az acs kubernetes get-credentials \ + --resource-group \ + --name + +If it asks for a password (to unlock the SSH key) +and you enter the correct password, +but you get an error message, +then try adding ``--ssh-key-file ~/.ssh/`` +to the above command (i.e. the path to the private key). + +.. note:: + + **About kubectl contexts.** You might manage several + Kubernetes clusters. To make it easy to switch from one to another, + kubectl has a notion of "contexts," e.g. the context for cluster 1 or + the context for cluster 2. To find out the current context, do: + + .. code:: bash + + $ kubectl config view + + and then look for the ``current-context`` in the output. + The output also lists all clusters, contexts and users. + (You might have only one of each.) + You can switch to a different context using: + + .. code:: bash + + $ kubectl config use-context + + You can also switch to a different context for just one command + by inserting ``--context `` into any kubectl command. + For example: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 get pods + + will get a list of the pods in the Kubernetes cluster associated + with the context named ``k8s-bdb-test-cluster-0``. + +Step 2: Connect to Your Cluster's Web UI (Optional) +--------------------------------------------------- + +You can connect to your cluster's +`Kubernetes Dashboard `_ +(also called the Web UI) using: + +.. code:: bash + + $ kubectl proxy -p 8001 + +or, if you prefer to be explicit about the context (explained above): + +.. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 proxy -p 8001 + +The output should be something like ``Starting to serve on 127.0.0.1:8001``. +That means you can visit the dashboard in your web browser at +`http://127.0.0.1:8001/ui `_. + + +Step 3: Configure Your BigchainDB Node +-------------------------------------- + +See the page titled :ref:`How to Configure a BigchainDB Node`. + + +Step 4: Start the NGINX Service +------------------------------- + + * This will will give us a public IP for the cluster. + + * Once you complete this step, you might need to wait up to 10 mins for the + public IP to be assigned. + + * You have the option to use vanilla NGINX without HTTPS support or an + OpenResty NGINX integrated with 3scale API Gateway. + + +Step 4.1: Vanilla NGINX +^^^^^^^^^^^^^^^^^^^^^^^ + + * This configuration is located in the file ``nginx/nginx-svc.yaml``. + + * Set the ``metadata.name`` and ``metadata.labels.name`` to the value + set in ``ngx-instance-name`` in the ConfigMap above. + + * Set the ``spec.selector.app`` to the value set in ``ngx-instance-name`` in + the ConfigMap followed by ``-dep``. For example, if the value set in the + ``ngx-instance-name`` is ``ngx-instance-0``, set the + ``spec.selector.app`` to ``ngx-instance-0-dep``. + + * Set ``ngx-public-mdb-port.port`` to 27017, or the port number on which you + want to expose MongoDB service. + Set the ``ngx-public-mdb-port.targetPort`` to the port number on which the + Kubernetes MongoDB service will be present. + + * Set ``ngx-public-api-port.port`` to 80, or the port number on which you want to + expose BigchainDB API service. + Set the ``ngx-public-api-port.targetPort`` to the port number on which the + Kubernetes BigchainDB API service will present. + + * Set ``ngx-public-ws-port.port`` to 81, or the port number on which you want to + expose BigchainDB Websocket service. + Set the ``ngx-public-ws-port.targetPort`` to the port number on which the + BigchainDB Websocket service will be present. + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f nginx/nginx-svc.yaml + + +Step 4.2: OpenResty NGINX + 3scale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * You have to enable HTTPS for this one and will need an HTTPS certificate + for your domain. + + * You should have already created the necessary Kubernetes Secrets in the previous + step (e.g. ``https-certs`` and ``threescale-credentials``). + + * This configuration is located in the file ``nginx-3scale/nginx-3scale-svc.yaml``. + + * Set the ``metadata.name`` and ``metadata.labels.name`` to the value + set in ``ngx-instance-name`` in the ConfigMap above. + + * Set the ``spec.selector.app`` to the value set in ``ngx-instance-name`` in + the ConfigMap followed by ``-dep``. For example, if the value set in the + ``ngx-instance-name`` is ``ngx-instance-0``, set the + ``spec.selector.app`` to ``ngx-instance-0-dep``. + + * Set ``ngx-public-mdb-port.port`` to 27017, or the port number on which you + want to expose MongoDB service. + Set the ``ngx-public-mdb-port.targetPort`` to the port number on which the + Kubernetes MongoDB service will be present. + + * Set ``ngx-public-3scale-port.port`` to 8080, or the port number on which + you want to let 3scale communicate with Openresty NGINX for authenctication. + Set the ``ngx-public-3scale-port.targetPort`` to the port number on which + this Openresty NGINX service will be listening to for communication with + 3scale. + + * Set ``ngx-public-bdb-port.port`` to 443, or the port number on which you want + to expose BigchainDB API service. + Set the ``ngx-public-api-port.targetPort`` to the port number on which the + Kubernetes BigchainDB API service will present. + + * Set ``ngx-public-bdb-port-http.port`` to 80, or the port number on which you + want to expose BigchainDB Websocket service. + Set the ``ngx-public-bdb-port-http.targetPort`` to the port number on which the + BigchainDB Websocket service will be present. + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-svc.yaml + + +Step 5: Assign DNS Name to the NGINX Public IP +---------------------------------------------- + + * This step is required only if you are planning to set up multiple + `BigchainDB nodes + `_ or are using + HTTPS certificates tied to a domain. + + * The following command can help you find out if the NGINX service started + above has been assigned a public IP or external IP address: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 get svc -w + + * Once a public IP is assigned, you can map it to + a DNS name. + We usually assign ``bdb-test-cluster-0``, ``bdb-test-cluster-1`` and + so on in our documentation. + Let's assume that we assign the unique name of ``bdb-test-cluster-0`` here. + + +**Set up DNS mapping in Azure.** +Select the current Azure resource group and look for the ``Public IP`` +resource. You should see at least 2 entries there - one for the Kubernetes +master and the other for the MongoDB instance. You may have to ``Refresh`` the +Azure web page listing the resources in a resource group for the latest +changes to be reflected. +Select the ``Public IP`` resource that is attached to your service (it should +have the Azure DNS prefix name along with a long random string, without the +``master-ip`` string), select ``Configuration``, add the DNS assigned above +(for example, ``bdb-test-cluster-0``), click ``Save``, and wait for the +changes to be applied. + +To verify the DNS setting is operational, you can run ``nslookup `` from your local Linux shell. + +This will ensure that when you scale the replica set later, other MongoDB +members in the replica set can reach this instance. + + +Step 6: Start the MongoDB Kubernetes Service +-------------------------------------------- + + * This configuration is located in the file ``mongodb/mongo-svc.yaml``. + + * Set the ``metadata.name`` and ``metadata.labels.name`` to the value + set in ``mdb-instance-name`` in the ConfigMap above. + + * Set the ``spec.selector.app`` to the value set in ``mdb-instance-name`` in + the ConfigMap followed by ``-ss``. For example, if the value set in the + ``mdb-instance-name`` is ``mdb-instance-0``, set the + ``spec.selector.app`` to ``mdb-instance-0-ss``. + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-svc.yaml + + +Step 7: Start the BigchainDB Kubernetes Service +----------------------------------------------- + + * This configuration is located in the file ``bigchaindb/bigchaindb-svc.yaml``. + + * Set the ``metadata.name`` and ``metadata.labels.name`` to the value + set in ``bdb-instance-name`` in the ConfigMap above. + + * Set the ``spec.selector.app`` to the value set in ``bdb-instance-name`` in + the ConfigMap followed by ``-dep``. For example, if the value set in the + ``bdb-instance-name`` is ``bdb-instance-0``, set the + ``spec.selector.app`` to ``bdb-instance-0-dep``. + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f bigchaindb/bigchaindb-svc.yaml + + +Step 8: Start the NGINX Kubernetes Deployment +--------------------------------------------- + + * NGINX is used as a proxy to both the BigchainDB and MongoDB instances in + the node. It proxies HTTP requests on port 80 to the BigchainDB backend, + and TCP connections on port 27017 to the MongoDB backend. + + * As in step 4, you have the option to use vanilla NGINX or an OpenResty + NGINX integrated with 3scale API Gateway. + +Step 8.1: Vanilla NGINX +^^^^^^^^^^^^^^^^^^^^^^^ + + * This configuration is located in the file ``nginx/nginx-dep.yaml``. + + * Set the ``metadata.name`` and ``spec.template.metadata.labels.app`` + to the value set in ``ngx-instance-name`` in the ConfigMap followed by a + ``-dep``. For example, if the value set in the ``ngx-instance-name`` is + ``ngx-instance-0``, set the fields to ``ngx-instance-0-dep``. + + * Set ``MONGODB_BACKEND_HOST`` env var to + the value set in ``mdb-instance-name`` in the ConfigMap, followed by + ``.default.svc.cluster.local``. For example, if the value set in the + ``mdb-instance-name`` is ``mdb-instance-0``, set the + ``MONGODB_BACKEND_HOST`` env var to + ``mdb-instance-0.default.svc.cluster.local``. + + * Set ``BIGCHAINDB_BACKEND_HOST`` env var to + the value set in ``bdb-instance-name`` in the ConfigMap, followed by + ``.default.svc.cluster.local``. For example, if the value set in the + ``bdb-instance-name`` is ``bdb-instance-0``, set the + ``BIGCHAINDB_BACKEND_HOST`` env var to + ``bdb-instance-0.default.svc.cluster.local``. + + * Start the Kubernetes Deployment: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f nginx/nginx-dep.yaml + + +Step 8.2: OpenResty NGINX + 3scale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * This configuration is located in the file + ``nginx-3scale/nginx-3scale-dep.yaml``. + + * Set the ``metadata.name`` and ``spec.template.metadata.labels.app`` + to the value set in ``ngx-instance-name`` in the ConfigMap followed by a + ``-dep``. For example, if the value set in the ``ngx-instance-name`` is + ``ngx-instance-0``, set the fields to ``ngx-instance-0-dep``. + + * Set ``MONGODB_BACKEND_HOST`` env var to + the value set in ``mdb-instance-name`` in the ConfigMap, followed by + ``.default.svc.cluster.local``. For example, if the value set in the + ``mdb-instance-name`` is ``mdb-instance-0``, set the + ``MONGODB_BACKEND_HOST`` env var to + ``mdb-instance-0.default.svc.cluster.local``. + + * Set ``BIGCHAINDB_BACKEND_HOST`` env var to + the value set in ``bdb-instance-name`` in the ConfigMap, followed by + ``.default.svc.cluster.local``. For example, if the value set in the + ``bdb-instance-name`` is ``bdb-instance-0``, set the + ``BIGCHAINDB_BACKEND_HOST`` env var to + ``bdb-instance-0.default.svc.cluster.local``. + + * Start the Kubernetes Deployment: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-dep.yaml + + +Step 9: Create Kubernetes Storage Classes for MongoDB +----------------------------------------------------- + +MongoDB needs somewhere to store its data persistently, +outside the container where MongoDB is running. +Our MongoDB Docker container +(based on the official MongoDB Docker container) +exports two volume mounts with correct +permissions from inside the container: + +* The directory where the mongod instance stores its data: ``/data/db``. + There's more explanation in the MongoDB docs about `storage.dbpath `_. + +* The directory where the mongodb instance stores the metadata for a sharded + cluster: ``/data/configdb/``. + There's more explanation in the MongoDB docs about `sharding.configDB `_. + +Explaining how Kubernetes handles persistent volumes, +and the associated terminology, +is beyond the scope of this documentation; +see `the Kubernetes docs about persistent volumes +`_. + +The first thing to do is create the Kubernetes storage classes. + +**Set up Storage Classes in Azure.** +First, you need an Azure storage account. +If you deployed your Kubernetes cluster on Azure +using the Azure CLI 2.0 +(as per :doc:`our template `), +then the `az acs create` command already created two +storage accounts in the same location and resource group +as your Kubernetes cluster. +Both should have the same "storage account SKU": ``Standard_LRS``. +Standard storage is lower-cost and lower-performance. +It uses hard disk drives (HDD). +LRS means locally-redundant storage: three replicas +in the same data center. +Premium storage is higher-cost and higher-performance. +It uses solid state drives (SSD). +At the time of writing, +when we created a storage account with SKU ``Premium_LRS`` +and tried to use that, +the PersistentVolumeClaim would get stuck in a "Pending" state. +For future reference, the command to create a storage account is +`az storage account create `_. + + +The Kubernetes template for configuration of Storage Class is located in the +file ``mongodb/mongo-sc.yaml``. + +You may have to update the ``parameters.location`` field in the file to +specify the location you are using in Azure. + +Create the required storage classes using: + +.. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-sc.yaml + + +You can check if it worked using ``kubectl get storageclasses``. + +**Azure.** Note that there is no line of the form +``storageAccount: `` +under ``parameters:``. When we included one +and then created a PersistentVolumeClaim based on it, +the PersistentVolumeClaim would get stuck +in a "Pending" state. +Kubernetes just looks for a storageAccount +with the specified skuName and location. + + +Step 10: Create Kubernetes Persistent Volume Claims +--------------------------------------------------- + +Next, you will create two PersistentVolumeClaim objects ``mongo-db-claim`` and +``mongo-configdb-claim``. + +This configuration is located in the file ``mongodb/mongo-pvc.yaml``. + +Note how there's no explicit mention of Azure, AWS or whatever. +``ReadWriteOnce`` (RWO) means the volume can be mounted as +read-write by a single Kubernetes node. +(``ReadWriteOnce`` is the *only* access mode supported +by AzureDisk.) +``storage: 20Gi`` means the volume has a size of 20 +`gibibytes `_. + +You may want to update the ``spec.resources.requests.storage`` field in both +the files to specify a different disk size. + +Create the required Persistent Volume Claims using: + +.. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-pvc.yaml + + +You can check its status using: ``kubectl get pvc -w`` + +Initially, the status of persistent volume claims might be "Pending" +but it should become "Bound" fairly quickly. + + +Step 11: Start a Kubernetes StatefulSet for MongoDB +--------------------------------------------------- + + * This configuration is located in the file ``mongodb/mongo-ss.yaml``. + + * Set the ``spec.serviceName`` to the value set in ``mdb-instance-name`` in + the ConfigMap. + For example, if the value set in the ``mdb-instance-name`` + is ``mdb-instance-0``, set the field to ``mdb-instance-0``. + + * Set ``metadata.name``, ``spec.template.metadata.name`` and + ``spec.template.metadata.labels.app`` to the value set in + ``mdb-instance-name`` in the ConfigMap, followed by + ``-ss``. + For example, if the value set in the + ``mdb-instance-name`` is ``mdb-instance-0``, set the fields to the value + ``mdb-insance-0-ss``. + + * Note how the MongoDB container uses the ``mongo-db-claim`` and the + ``mongo-configdb-claim`` PersistentVolumeClaims for its ``/data/db`` and + ``/data/configdb`` directories (mount paths). + + * Note also that we use the pod's ``securityContext.capabilities.add`` + specification to add the ``FOWNER`` capability to the container. That is + because the MongoDB container has the user ``mongodb``, with uid ``999`` and + group ``mongodb``, with gid ``999``. + When this container runs on a host with a mounted disk, the writes fail + when there is no user with uid ``999``. To avoid this, we use the Docker + feature of ``--cap-add=FOWNER``. This bypasses the uid and gid permission + checks during writes and allows data to be persisted to disk. + Refer to the `Docker docs + `_ + for details. + + * As we gain more experience running MongoDB in testing and production, we + will tweak the ``resources.limits.cpu`` and ``resources.limits.memory``. + + * Create the MongoDB StatefulSet using: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-ss.yaml + + * It might take up to 10 minutes for the disks, specified in the Persistent + Volume Claims above, to be created and attached to the pod. + The UI might show that the pod has errored with the message + "timeout expired waiting for volumes to attach/mount". Use the CLI below + to check the status of the pod in this case, instead of the UI. + This happens due to a bug in Azure ACS. + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 get pods -w + + +Step 12: Configure Users and Access Control for MongoDB +------------------------------------------------------- + + * In this step, you will create a user on MongoDB with authorization + to create more users and assign + roles to them. + Note: You need to do this only when setting up the first MongoDB node of + the cluster. + + * Find out the name of your MongoDB pod by reading the output + of the ``kubectl ... get pods`` command at the end of the last step. + It should be something like ``mdb-instance-0-ss-0``. + + * Log in to the MongoDB pod using: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 exec -it bash + + * Open a mongo shell using the certificates + already present at ``/etc/mongod/ssl/`` + + .. code:: bash + + $ mongo --host localhost --port 27017 --verbose --ssl \ + --sslCAFile /etc/mongod/ssl/ca.pem \ + --sslPEMKeyFile /etc/mongod/ssl/mdb-instance.pem + + * Initialize the replica set using: + + .. code:: bash + + > rs.initiate( { + _id : "bigchain-rs", + members: [ { + _id : 0, + host :":27017" + } ] + } ) + + The ``hostname`` in this case will be the value set in + ``mdb-instance-name`` in the ConfigMap. + For example, if the value set in the ``mdb-instance-name`` is + ``mdb-instance-0``, set the ``hostname`` above to the value ``mdb-instance-0``. + + * The instance should be voted as the ``PRIMARY`` in the replica set (since + this is the only instance in the replica set till now). + This can be observed from the mongo shell prompt, + which will read ``PRIMARY>``. + + * Create a user ``adminUser`` on the ``admin`` database with the + authorization to create other users. This will only work the first time you + log in to the mongo shell. For further details, see `localhost + exception `_ + in MongoDB. + + .. code:: bash + + PRIMARY> use admin + PRIMARY> db.createUser( { + user: "adminUser", + pwd: "superstrongpassword", + roles: [ { role: "userAdminAnyDatabase", db: "admin" } ] + } ) + + * Exit and restart the mongo shell using the above command. + Authenticate as the ``adminUser`` we created earlier: + + .. code:: bash + + PRIMARY> use admin + PRIMARY> db.auth("adminUser", "superstrongpassword") + + ``db.auth()`` returns 0 when authentication is not successful, + and 1 when successful. + + * We need to specify the user name *as seen in the certificate* issued to + the BigchainDB instance in order to authenticate correctly. Use + the following ``openssl`` command to extract the user name from the + certificate: + + .. code:: bash + + $ openssl x509 -in \ + -inform PEM -subject -nameopt RFC2253 + + You should see an output line that resembles: + + .. code:: bash + + subject= emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE + + The ``subject`` line states the complete user name we need to use for + creating the user on the mongo shell as follows: + + .. code:: bash + + PRIMARY> db.getSiblingDB("$external").runCommand( { + createUser: 'emailAddress=dev@bigchaindb.com,CN=test-bdb-ssl,OU=BigchainDB-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE', + writeConcern: { w: 'majority' , wtimeout: 5000 }, + roles: [ + { role: 'clusterAdmin', db: 'admin' }, + { role: 'readWriteAnyDatabase', db: 'admin' } + ] + } ) + + * You can similarly create users for MongoDB Monitoring Agent and MongoDB + Backup Agent. For example: + + .. code:: bash + + PRIMARY> db.getSiblingDB("$external").runCommand( { + createUser: 'emailAddress=dev@bigchaindb.com,CN=test-mdb-mon-ssl,OU=MongoDB-Mon-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE', + writeConcern: { w: 'majority' , wtimeout: 5000 }, + roles: [ + { role: 'clusterMonitor', db: 'admin' } + ] + } ) + + PRIMARY> db.getSiblingDB("$external").runCommand( { + createUser: 'emailAddress=dev@bigchaindb.com,CN=test-mdb-bak-ssl,OU=MongoDB-Bak-Instance,O=BigchainDB GmbH,L=Berlin,ST=Berlin,C=DE', + writeConcern: { w: 'majority' , wtimeout: 5000 }, + roles: [ + { role: 'backup', db: 'admin' } + ] + } ) + + +Step 13: Start a Kubernetes Deployment for MongoDB Monitoring Agent +------------------------------------------------------------------- + + * This configuration is located in the file + ``mongodb-monitoring-agent/mongo-mon-dep.yaml``. + + * Set ``metadata.name``, ``spec.template.metadata.name`` and + ``spec.template.metadata.labels.app`` to the value set in + ``mdb-mon-instance-name`` in the ConfigMap, followed by + ``-dep``. + For example, if the value set in the + ``mdb-mon-instance-name`` is ``mdb-mon-instance-0``, set the fields to the + value ``mdb-mon-instance-0-dep``. + + * Start the Kubernetes Deployment using: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb-monitoring-agent/mongo-mon-dep.yaml + + +Step 14: Start a Kubernetes Deployment for MongoDB Backup Agent +--------------------------------------------------------------- + + * This configuration is located in the file + ``mongodb-backup-agent/mongo-backup-dep.yaml``. + + * Set ``metadata.name``, ``spec.template.metadata.name`` and + ``spec.template.metadata.labels.app`` to the value set in + ``mdb-bak-instance-name`` in the ConfigMap, followed by + ``-dep``. + For example, if the value set in the + ``mdb-bak-instance-name`` is ``mdb-bak-instance-0``, set the fields to the + value ``mdb-bak-instance-0-dep``. + + * Start the Kubernetes Deployment using: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb-backup-agent/mongo-backup-dep.yaml + + +Step 15: Start a Kubernetes Deployment for BigchainDB +----------------------------------------------------- + + * This configuration is located in the file + ``bigchaindb/bigchaindb-dep.yaml``. + + * Set ``metadata.name`` and ``spec.template.metadata.labels.app`` to the + value set in ``bdb-instance-name`` in the ConfigMap, followed by + ``-dep``. + For example, if the value set in the + ``bdb-instance-name`` is ``bdb-instance-0``, set the fields to the + value ``bdb-insance-0-dep``. + + * Set the value of ``BIGCHAINDB_KEYPAIR_PRIVATE`` (not base64-encoded). + (In the future, we'd like to pull the BigchainDB private key from + the Secret named ``bdb-private-key``, + but a Secret can only be mounted as a file, + so BigchainDB Server would have to be modified to look for it + in a file.) + + * As we gain more experience running BigchainDB in testing and production, + we will tweak the ``resources.limits`` values for CPU and memory, and as + richer monitoring and probing becomes available in BigchainDB, we will + tweak the ``livenessProbe`` and ``readinessProbe`` parameters. + + * Create the BigchainDB Deployment using: + + .. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 apply -f bigchaindb/bigchaindb-dep.yaml + + + * You can check its status using the command ``kubectl get deployments -w`` + + +Step 16: Configure the MongoDB Cloud Manager +-------------------------------------------- + +Refer to the +:ref:`documentation ` +for details on how to configure the MongoDB Cloud Manager to enable +monitoring and backup. + + +Step 17: Verify the BigchainDB Node Setup +----------------------------------------- + +Step 17.1: Testing Internally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To test the setup of your BigchainDB node, you could use a Docker container +that provides utilities like ``nslookup``, ``curl`` and ``dig``. +For example, you could use a container based on our +`bigchaindb/toolbox `_ image. +(The corresponding +`Dockerfile `_ +is in the ``bigchaindb/bigchaindb`` repository on GitHub.) +You can use it as below to get started immediately: + +.. code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 \ + run -it toolbox \ + --image bigchaindb/toolbox \ + --image-pull-policy=Always \ + --restart=Never --rm + +It will drop you to the shell prompt. + +To test the MongoDB instance: + +.. code:: bash + + $ nslookup mdb-instance-0 + + $ dig +noall +answer _mdb-port._tcp.mdb-instance-0.default.svc.cluster.local SRV + + $ curl -X GET http://mdb-instance-0:27017 + +The ``nslookup`` command should output the configured IP address of the service +(in the cluster). +The ``dig`` command should return the configured port numbers. +The ``curl`` command tests the availability of the service. + +To test the BigchainDB instance: + +.. code:: bash + + $ nslookup bdb-instance-0 + + $ dig +noall +answer _bdb-port._tcp.bdb-instance-0.default.svc.cluster.local SRV + + $ curl -X GET http://bdb-instance-0:9984 + +To test the NGINX instance: + +.. code:: bash + + $ nslookup ngx-instance-0 + + $ dig +noall +answer _ngx-public-mdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + $ dig +noall +answer _ngx-public-bdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + $ curl -X GET http://ngx-instance-0:27017 + +The curl command should result get the response +``curl: (7) Failed to connect to ngx-instance-0 port 27017: Connection refused``. + +If you ran the vanilla NGINX instance, run: + +.. code:: bash + + $ curl -X GET http://ngx-instance-0:80 + +If you ran the OpenResty NGINX + 3scale instance, run: + +.. code:: bash + + $ curl -X GET https://ngx-instance-0 + + +Step 17.2: Testing Externally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Check the MongoDB monitoring and backup agent on the MongoDB Cloud Manager +portal to verify they are working fine. + +Try to access the ``:80`` +on your browser. You should receive a JSON response that shows the BigchainDB +server version, among other things. + +Use the Python Driver to send some transactions to the BigchainDB node and +verify that your node or cluster works as expected. diff --git a/docs/server/source/cloud-deployment-templates/revoke-tls-certificate.rst b/docs/server/source/production-deployment-template/revoke-tls-certificate.rst similarity index 83% rename from docs/server/source/cloud-deployment-templates/revoke-tls-certificate.rst rename to docs/server/source/production-deployment-template/revoke-tls-certificate.rst index 5c566c97..7584ceb5 100644 --- a/docs/server/source/cloud-deployment-templates/revoke-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/revoke-tls-certificate.rst @@ -1,8 +1,8 @@ How to Revoke an SSL/TLS Certificate ==================================== -This page enumerates the steps *we* take to revoke a self-signed SSL/TLS certificate -in a cluster. +This page enumerates the steps *we* take to revoke a self-signed SSL/TLS +certificate in a cluster. It can only be done by someone with access to the self-signed CA associated with the cluster's managing organization. @@ -23,11 +23,11 @@ certificate: ./easyrsa revoke + This will update the CA database with the revocation details. The next step is to use the updated database to issue an up-to-date certificate revocation list (CRL). - Step 2: Generate a New CRL -------------------------- @@ -40,3 +40,4 @@ Generate a new CRL for your infrastructure using: The generated ``crl.pem`` file needs to be uploaded to your infrastructure to prevent the revoked certificate from being used again. +In particlar, the generated ``crl.pem`` file should be sent to all BigchainDB node operators in your BigchainDB cluster, so that they can update it in their MongoDB instance and their BigchainDB Server instance. diff --git a/docs/server/source/cloud-deployment-templates/server-tls-certificate.rst b/docs/server/source/production-deployment-template/server-tls-certificate.rst similarity index 69% rename from docs/server/source/cloud-deployment-templates/server-tls-certificate.rst rename to docs/server/source/production-deployment-template/server-tls-certificate.rst index b9cb1a14..eb9dd204 100644 --- a/docs/server/source/cloud-deployment-templates/server-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/server-tls-certificate.rst @@ -26,7 +26,7 @@ Step 2: Create the Server Private Key and CSR --------------------------------------------- You can create the server private key and certificate signing request (CSR) -by going into the directory ``member-cert/easy-rsa-3.0.1/easyrsa`` +by going into the directory ``member-cert/easy-rsa-3.0.1/easyrsa3`` and using something like: .. code:: bash @@ -35,32 +35,36 @@ and using something like: ./easyrsa --req-cn=mdb-instance-0 --subject-alt-name=DNS:localhost,DNS:mdb-instance-0 gen-req mdb-instance-0 nopass -You must replace the common name (``mdb-instance-0`` above) -with the common name of *your* MongoDB instance -(which should be the same as the hostname of your MongoDB instance). +You should replace the Common Name (``mdb-instance-0`` above) with the correct name for *your* MongoDB instance in the cluster, e.g. ``mdb-instance-5`` or ``mdb-instance-12``. (This name is decided by the organization managing the cluster.) -You need to provide the ``DNS:localhost`` SAN during certificate generation for -using the ``localhost exception`` in the MongoDB instance. +You will be prompted to enter the Distinguished Name (DN) information for this certificate. +For each field, you can accept the default value [in brackets] by pressing Enter. +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value ``MongoDB-Instance``. + +Aside: You need to provide the ``DNS:localhost`` SAN during certificate generation +for using the ``localhost exception`` in the MongoDB instance. All certificates can have this attribute without compromising security as the ``localhost exception`` works only the first time. -Tip: You can get help with the ``easyrsa`` command (and its subcommands) -by using the subcommand ``./easyrsa help`` - Step 3: Get the Server Certificate Signed ----------------------------------------- -The CSR file (created in the last step) -should be located in ``pki/reqs/mdb-instance-0.req``. +The CSR file created in the last step +should be located in ``pki/reqs/mdb-instance-0.req`` +(where the integer ``0`` may be different for you). You need to send it to the organization managing the cluster so that they can use their CA to sign the request. (The managing organization should already have a self-signed CA.) If you are the admin of the managing organization's self-signed CA, -then you can import the CSR and use Easy-RSA to sign it. For example: +then you can import the CSR and use Easy-RSA to sign it. +Go to your ``bdb-cluster-ca/easy-rsa-3.0.1/easyrsa3/`` +directory and do something like: .. code:: bash @@ -83,10 +87,3 @@ private keys. cat mdb-instance-0.crt mdb-instance-0.key > mdb-instance-0.pem - -Step 5: Update the MongoDB Config File --------------------------------------- - -In the MongoDB configuration file, -set the ``net.ssl.PEMKeyFile`` parameter to the path of the ``mdb-instance-0.pem`` file, -and the ``net.ssl.CAFile`` parameter to the ``ca.crt`` file. diff --git a/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst similarity index 92% rename from docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst rename to docs/server/source/production-deployment-template/template-kubernetes-azure.rst index a9e6792c..d99596cc 100644 --- a/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst +++ b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst @@ -45,11 +45,12 @@ on most common operating systems `_. Do that. -First, update the Azure CLI to the latest version: +If you already *have* the Azure CLI installed, you may want to update it. -.. code:: bash +.. warning:: + + ``az component update`` isn't supported if you installed the CLI using some of Microsoft's provided installation instructions. See `the Microsoft docs for update instructions `_. - $ az component update Next, login to your account using: @@ -101,7 +102,7 @@ Finally, you can deploy an ACS using something like: --agent-vm-size Standard_D2_v2 \ --dns-prefix \ --ssh-key-value ~/.ssh/.pub \ - --orchestrator-type kubernetes + --orchestrator-type kubernetes \ --debug --output json @@ -138,7 +139,7 @@ of a master node from the Azure Portal. For example: .. note:: - All the master nodes should have the *same* IP address and hostname + All the master nodes should have the *same* public IP address and hostname (also called the Master FQDN). The "agent" nodes shouldn't get public IP addresses or hostnames, diff --git a/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst b/docs/server/source/production-deployment-template/upgrade-on-kubernetes.rst similarity index 100% rename from docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst rename to docs/server/source/production-deployment-template/upgrade-on-kubernetes.rst diff --git a/docs/server/source/cloud-deployment-templates/workflow.rst b/docs/server/source/production-deployment-template/workflow.rst similarity index 78% rename from docs/server/source/cloud-deployment-templates/workflow.rst rename to docs/server/source/production-deployment-template/workflow.rst index b8aa919f..c511b8f9 100644 --- a/docs/server/source/cloud-deployment-templates/workflow.rst +++ b/docs/server/source/production-deployment-template/workflow.rst @@ -53,6 +53,26 @@ Similarly, other instances must also have unique names in the cluster. #. Name of the MongoDB backup agent instance (``mdb-bak-instance-*``) +☐ Generate four keys and corresponding certificate signing requests (CSRs): + +#. Server Certificate (a.k.a. Member Certificate) for the MongoDB instance +#. Client Certificate for BigchainDB Server to identify itself to MongoDB +#. Client Certificate for MongoDB Monitoring Agent to identify itself to MongoDB +#. Client Certificate for MongoDB Backup Agent to identify itself to MongoDB + +Ask the managing organization to use its self-signed CA to sign those four CSRs. +They should send you: + +* Four certificates (one for each CSR you sent them). +* One ``ca.crt`` file: their CA certificate. +* One ``crl.pem`` file: a certificate revocation list. + +For help, see the pages: + +* :ref:`How to Generate a Server Certificate for MongoDB` +* :ref:`How to Generate a Client Certificate for MongoDB` + + ☐ Every node in a BigchainDB cluster needs its own BigchainDB keypair (i.e. a public key and corresponding private key). You can generate a BigchainDB keypair for your node, for example, @@ -73,15 +93,17 @@ Don't share your private key. That list of public keys is known as the BigchainDB "keyring." -☐ Ask the managing organization -for the FQDN used to serve the BigchainDB APIs -(e.g. ``api.orgname.net`` or ``bdb.clustername.com``). - - ☐ Make up an FQDN for your BigchainDB node (e.g. ``mynode.mycorp.com``). Make sure you've registered the associated domain name (e.g. ``mycorp.com``), and have an SSL certificate for the FQDN. -(You can get an SSL certificate from any SSL certificate provider). +(You can get an SSL certificate from any SSL certificate provider.) + + +☐ Ask the managing organization +for the FQDN used to serve the BigchainDB APIs +(e.g. ``api.orgname.net`` or ``bdb.clustername.com``) +and for a copy of the associated SSL/TLS certificate. +Also, ask for the user name to use for authenticating to MongoDB. ☐ If the cluster uses 3scale for API authentication, monitoring and billing, @@ -89,35 +111,20 @@ you must ask the managing organization for all relevant 3scale credentials. ☐ If the cluster uses MongoDB Cloud Manager for monitoring and backup, -you must ask the managing organization for the ``Agent Api Key``. -(Each Cloud Manager backup will have its own ``Agent Api Key``. -If there's one Cloud Manager backup, -there will be one ``Agent Api Key`` for the whole cluster.) - - -☐ Generate four keys and corresponding certificate signing requests (CSRs): - -#. Server Certificate (a.k.a. Member Certificate) for the MongoDB instance -#. Client Certificate for BigchainDB Server to identify itself to MongoDB -#. Client Certificate for MongoDB Monitoring Agent to identify itself to MongoDB -#. Client Certificate for MongoDB Backup Agent to identify itself to MongoDB - -Ask the managing organization to use its self-signed CA to sign those certificates. - -For help, see the pages: - -* :ref:`How to Generate a Server Certificate for MongoDB` -* :ref:`How to Generate a Client Certificate for MongoDB` +you must ask the managing organization for the ``Group ID`` and the +``Agent API Key``. +(Each Cloud Manager "group" has its own ``Group ID``. A ``Group ID`` can +contain a number of ``Agent API Key`` s. It can be found under +**Settings - Group Settings**. It was recently added to the Cloud Manager to +allow easier periodic rotation of the ``Agent API Key`` with a constant +``Group ID``) ☐ :doc:`Deploy a Kubernetes cluster on Azure `. -☐ Create the Kubernetes Configuration for this node. -We will use Kubernetes ConfigMaps and Secrets to hold all the information -gathered above. - - -☐ Deploy your BigchainDB node on your Kubernetes cluster. - -TODO: Links to instructions for first-node-in-cluster or second-or-later-node-in-cluster \ No newline at end of file +☐ You can now proceed to set up your BigchainDB node based on whether it is the +:ref:`first node in a new cluster +` or a +:ref:`node that will be added to an existing cluster +`. diff --git a/docs/server/source/production-nodes/node-assumptions.md b/docs/server/source/production-nodes/node-assumptions.md index 9d52aa5a..136b6415 100644 --- a/docs/server/source/production-nodes/node-assumptions.md +++ b/docs/server/source/production-nodes/node-assumptions.md @@ -13,4 +13,4 @@ We make some assumptions about production nodes: You can use RethinkDB when building prototypes, but we don't advise or support using it in production. -We don't provide a detailed cookbook explaining how to secure a server, or other things that a sysadmin should know. (We do provide some [templates](../cloud-deployment-templates/index.html), but those are just a starting point.) +We don't provide a detailed cookbook explaining how to secure a server, or other things that a sysadmin should know. We do provide some templates, but those are just starting points. diff --git a/docs/server/source/production-nodes/node-requirements.md b/docs/server/source/production-nodes/node-requirements.md index 9588747b..d3504af9 100644 --- a/docs/server/source/production-nodes/node-requirements.md +++ b/docs/server/source/production-nodes/node-requirements.md @@ -5,9 +5,9 @@ ## OS Requirements -BigchainDB Server requires Python 3.4+ and Python 3.4+ [will run on any modern OS](https://docs.python.org/3.4/using/index.html), but we recommend using an LTS version of [Ubuntu Server](https://www.ubuntu.com/server) or a similarly server-grade Linux distribution. +BigchainDB Server requires Python 3.5+ and Python 3.5+ [will run on any modern OS](https://docs.python.org/3.5/using/index.html), but we recommend using an LTS version of [Ubuntu Server](https://www.ubuntu.com/server) or a similarly server-grade Linux distribution. -_Don't use macOS_ (formerly OS X, formerly Mac OS X), because it's not a server-grade operating system. Also, BigchaindB Server uses the Python multiprocessing package and [some functionality in the multiprocessing package doesn't work on Mac OS X](https://docs.python.org/3.4/library/multiprocessing.html#multiprocessing.Queue.qsize). +_Don't use macOS_ (formerly OS X, formerly Mac OS X), because it's not a server-grade operating system. Also, BigchaindB Server uses the Python multiprocessing package and [some functionality in the multiprocessing package doesn't work on Mac OS X](https://docs.python.org/3.5/library/multiprocessing.html#multiprocessing.Queue.qsize). ## General Considerations diff --git a/docs/server/source/production-nodes/setup-run-node.md b/docs/server/source/production-nodes/setup-run-node.md index 6e7ddbea..581e0cfe 100644 --- a/docs/server/source/production-nodes/setup-run-node.md +++ b/docs/server/source/production-nodes/setup-run-node.md @@ -2,7 +2,7 @@ This is a page of general guidelines for setting up a production BigchainDB node. Before continuing, make sure you've read the pages about production node [assumptions](node-assumptions.html), [components](node-components.html) and [requirements](node-requirements.html). -Note: These are just guidelines. You can modify them to suit your needs. For example, if you want to initialize the MongoDB replica set before installing BigchainDB, you _can_ do that. If you'd prefer to use Docker and Kubernetes, you can (and [we have a template](../cloud-deployment-templates/node-on-kubernetes.html)). We don't cover all possible setup procedures here. +Note: These are just guidelines. You can modify them to suit your needs. For example, if you want to initialize the MongoDB replica set before installing BigchainDB, you _can_ do that. If you'd prefer to use Docker and Kubernetes, you can (and [we have a template](../production-deployment-template/index.html)). We don't cover all possible setup procedures here. ## Security Guidelines @@ -50,16 +50,16 @@ Consult the MongoDB documentation for its recommendations regarding storage hard ### Install BigchainDB Server Dependencies -Before you can install BigchainDB Server, you must [install its OS-level dependencies](../appendices/install-os-level-deps.html) and you may have to [install Python 3.4+](https://www.python.org/downloads/). +Before you can install BigchainDB Server, you must [install its OS-level dependencies](../appendices/install-os-level-deps.html) and you may have to [install Python 3.5+](https://www.python.org/downloads/). ### How to Install BigchainDB Server with pip -BigchainDB is distributed as a Python package on PyPI so you can install it using `pip`. First, make sure you have an up-to-date Python 3.4+ version of `pip` installed: +BigchainDB is distributed as a Python package on PyPI so you can install it using `pip`. First, make sure you have an up-to-date Python 3.5+ version of `pip` installed: ```text pip -V ``` -If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.4, then you must install a `pip` version associated with Python 3.4+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). +If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.5, then you must install a `pip` version associated with Python 3.5+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). On Ubuntu 16.04, we found that this works: ```text diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 5c2b0500..d31f06ae 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -4,7 +4,7 @@ This page has instructions to set up a single stand-alone BigchainDB node for le A. Install MongoDB as the database backend. (There are other options but you can ignore them for now.) -[Install MongoDB Server 3.4+](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) +[Install MongoDB Server 3.5+](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) B. Run MongoDB. Open a Terminal and run the command: ```text diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 3cb62c41..94e5bf70 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -21,7 +21,7 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_SERVER_BIND`
`BIGCHAINDB_SERVER_LOGLEVEL`
`BIGCHAINDB_SERVER_WORKERS`
-`BIGCHAINDB_SERVER_THREADS`
+`BIGCHAINDB_WSSERVER_SCHEME`
`BIGCHAINDB_WSSERVER_HOST`
`BIGCHAINDB_WSSERVER_PORT`
`BIGCHAINDB_CONFIG_PATH`
@@ -36,6 +36,15 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_LOG_FMT_CONSOLE`
`BIGCHAINDB_LOG_FMT_LOGFILE`
`BIGCHAINDB_LOG_GRANULAR_LEVELS`
+`BIGCHAINDB_DATABASE_SSL`
+`BIGCHIANDB_DATABASE_LOGIN`
+`BIGCHAINDB_DATABASE_PASSWORD`
+`BIGCHAINDB_DATABASE_CA_CERT`
+`BIGCHAINDB_DATABASE_CERTFILE`
+`BIGCHAINDB_DATABASE_KEYFILE`
+`BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE`
+`BIGCHAINDB_DATABASE_CRLFILE`
+`BIGCHAINDB_GRAPHITE_HOST`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -43,7 +52,7 @@ Note that the `-c` command line option will always take precedence if both the ` You can read the current default values in the file [bigchaindb/\_\_init\_\_.py](https://github.com/bigchaindb/bigchaindb/blob/master/bigchaindb/__init__.py). (The link is to the latest version.) -Running `bigchaindb -y configure rethinkdb` will generate a local config file in `$HOME/.bigchaindb` with all the default values, with two exceptions: It will generate a valid private/public keypair, rather than using the default keypair (`None` and `None`). +Running `bigchaindb -y configure mongodb` will generate a local config file in `$HOME/.bigchaindb` with all the default values (for using MongoDB as the database backend), with two exceptions: it will generate a valid private/public keypair, rather than using the default keypair (`None` and `None`). ## keypair.public & keypair.private @@ -64,7 +73,7 @@ export BIGCHAINDB_KEYPAIR_PRIVATE=5C5Cknco7YxBRP9AgB1cbUVTL4FAcooxErLygw1DeG2D } ``` -Internally (i.e. in the Python code), both keys have a default value of `None`, but that's not a valid key. Therefore you can't rely on the defaults for the keypair. If you want to run BigchainDB, you must provide a valid keypair, either in the environment variables or in the local config file. You can generate a local config file with a valid keypair (and default everything else) using `bigchaindb -y configure rethinkdb`. +Internally (i.e. in the Python code), both keys have a default value of `None`, but that's not a valid key. Therefore you can't rely on the defaults for the keypair. If you want to run BigchainDB, you must provide a valid keypair, either in the environment variables or in the local config file. You can generate a local config file with a valid keypair (and default everything else) using `bigchaindb -y configure mongodb`. ## keyring @@ -93,15 +102,28 @@ Note how the keys in the list are separated by colons. ## database.* The settings with names of the form `database.*` are for the database backend -(currently either RethinkDB or MongoDB). They are: +(currently either MongoDB or RethinkDB). They are: -* `database.backend` is either `rethinkdb` or `mongodb`. +* `database.backend` is either `mongodb` or `rethinkdb`. * `database.host` is the hostname (FQDN) of the backend database. * `database.port` is self-explanatory. -* `database.name` is a user-chosen name for the database inside RethinkDB or MongoDB, e.g. `bigchain`. +* `database.name` is a user-chosen name for the database inside MongoDB or RethinkDB, e.g. `bigchain`. * `database.replicaset` is only relevant if using MongoDB; it's the name of the MongoDB replica set, e.g. `bigchain-rs`. * `database.connection_timeout` is the maximum number of milliseconds that BigchainDB will wait before giving up on one attempt to connect to the database backend. * `database.max_tries` is the maximum number of times that BigchainDB will try to establish a connection with the database backend. If 0, then it will try forever. +* `database.ssl` is a flag that determines if BigchainDB connects to the + backend database over TLS/SSL or not. This can be set to either `true` or + `false` (the default). + Note: This parameter is only supported for the MongoDB backend currently. +* `database.login` and `database.password` are the login and password used to + authenticate to the database before performing any operations, specified in + plaintext. The default values for both are currently `null`, which means that + BigchainDB will not authenticate with the backend database. + Note: These parameters are only supported for the MongoDB backend currently. +* `database.ca_cert`, `database.certfile`, `database.keyfile` and `database.crlfile` are the paths to the CA, signed certificate, private key and certificate revocation list files respectively. + Note: These parameters are only supported for the MongoDB backend currently. +* `database.keyfile_passphrase` is the private key decryption passphrase, specified in plaintext. + Note: This parameter is only supported for the MongoDB backend currently. **Example using environment variables** ```text @@ -137,7 +159,15 @@ If you used `bigchaindb -y configure mongodb` to create a default local config f "name": "bigchain", "replicaset": "bigchain-rs", "connection_timeout": 5000, - "max_tries": 3 + "max_tries": 3, + "login": null, + "password": null + "ssl": false, + "ca_cert": null, + "crlfile": null, + "certfile": null, + "keyfile": null, + "keyfile_passphrase": null, } ``` @@ -146,20 +176,19 @@ If you used `bigchaindb -y configure mongodb` to create a default local config f These settings are for the [Gunicorn HTTP server](http://gunicorn.org/), which is used to serve the [HTTP client-server API](../http-client-server-api.html). -`server.bind` is where to bind the Gunicorn HTTP server socket. It's a string. It can be any valid value for [Gunicorn's bind setting](http://docs.gunicorn.org/en/stable/settings.html#bind). If you want to allow IPv4 connections from anyone, on port 9984, use '0.0.0.0:9984'. In a production setting, we recommend you use Gunicorn behind a reverse proxy server. If Gunicorn and the reverse proxy are running on the same machine, then use 'localhost:PORT' where PORT is _not_ 9984 (because the reverse proxy needs to listen on port 9984). Maybe use PORT=9983 in that case because we know 9983 isn't used. If Gunicorn and the reverse proxy are running on different machines, then use 'A.B.C.D:9984' where A.B.C.D is the IP address of the reverse proxy. There's [more information about deploying behind a reverse proxy in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) +`server.bind` is where to bind the Gunicorn HTTP server socket. It's a string. It can be any valid value for [Gunicorn's bind setting](http://docs.gunicorn.org/en/stable/settings.html#bind). If you want to allow IPv4 connections from anyone, on port 9984, use `0.0.0.0:9984`. In a production setting, we recommend you use Gunicorn behind a reverse proxy server. If Gunicorn and the reverse proxy are running on the same machine, then use `localhost:PORT` where PORT is _not_ 9984 (because the reverse proxy needs to listen on port 9984). Maybe use PORT=9983 in that case because we know 9983 isn't used. If Gunicorn and the reverse proxy are running on different machines, then use `A.B.C.D:9984` where A.B.C.D is the IP address of the reverse proxy. There's [more information about deploying behind a reverse proxy in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) `server.loglevel` sets the log level of Gunicorn's Error log outputs. See [Gunicorn's documentation](http://docs.gunicorn.org/en/latest/settings.html#loglevel) for more information. -`server.workers` is [the number of worker processes](http://docs.gunicorn.org/en/stable/settings.html#workers) for handling requests. If `None` (the default), the value will be (cpu_count * 2 + 1). Each worker process has a single thread. The HTTP server will be able to handle `server.workers` requests simultaneously. +`server.workers` is [the number of worker processes](http://docs.gunicorn.org/en/stable/settings.html#workers) for handling requests. If `None` (the default), the value will be (2 × cpu_count + 1). Each worker process has a single thread. The HTTP server will be able to handle `server.workers` requests simultaneously. **Example using environment variables** ```text export BIGCHAINDB_SERVER_BIND=0.0.0.0:9984 export BIGCHAINDB_SERVER_LOGLEVEL=debug export BIGCHAINDB_SERVER_WORKERS=5 -export BIGCHAINDB_SERVER_THREADS=5 ``` **Example config file snippet** @@ -181,12 +210,14 @@ export BIGCHAINDB_SERVER_THREADS=5 ``` -## wsserver.host and wsserver.port +## wsserver.scheme, wsserver.host and wsserver.port These settings are for the [aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), which is used to serve the [WebSocket Event Stream API](../websocket-event-stream-api.html). +`wsserver.scheme` should be either `"ws"` or `"wss"` +(but setting it to `"wss"` does *not* enable SSL/TLS). `wsserver.host` is where to bind the aiohttp server socket and `wsserver.port` is the corresponding port. If you want to allow connections from anyone, on port 9985, @@ -194,6 +225,7 @@ set `wsserver.host` to 0.0.0.0 and `wsserver.port` to 9985. **Example using environment variables** ```text +export BIGCHAINDB_WSSERVER_SCHEME=ws export BIGCHAINDB_WSSERVER_HOST=0.0.0.0 export BIGCHAINDB_WSSERVER_PORT=9985 ``` @@ -201,6 +233,7 @@ export BIGCHAINDB_WSSERVER_PORT=9985 **Example config file snippet** ```js "wsserver": { + "scheme": "wss", "host": "0.0.0.0", "port": 65000 } @@ -209,6 +242,7 @@ export BIGCHAINDB_WSSERVER_PORT=9985 **Default values (from a config file)** ```js "wsserver": { + "scheme": "ws", "host": "localhost", "port": 9985 } @@ -462,3 +496,29 @@ logging of the `core.py` module to be more verbose, you would set the ``` **Defaults to**: `"{}"` + + +## graphite.host + +The host name or IP address of a server listening for statsd events on UDP +port 8125. This defaults to `localhost`, and if no statsd collector is running, +the events are simply dropped by the operating system. + +**Example using environment variables** +```text +export BIGCHAINDB_GRAPHITE_HOST=10.0.0.5 +``` + +**Example config file snippet** +```js +"graphite": { + "host": "10.0.0.5" +} +``` + +**Default values (from a config file)** +```js +"graphite": { + "host": "localhost" +} +``` diff --git a/docs/server/source/websocket-event-stream-api.rst b/docs/server/source/websocket-event-stream-api.rst index 0310ef63..0000107b 100644 --- a/docs/server/source/websocket-event-stream-api.rst +++ b/docs/server/source/websocket-event-stream-api.rst @@ -26,16 +26,14 @@ It's a good idea to make sure that the node you're connecting with has advertised support for the Event Stream API. To do so, send a HTTP GET request to the node's :ref:`API Root Endpoint` (e.g. ``http://localhost:9984/api/v1/``) and check that the -response contains a ``streams_`` property in ``_links``: +response contains a ``streams`` property: .. code:: JSON { - "_links": { - ..., - "streams_v1": "ws://example.com:9985/api/v1/streams/valid_tx", - ... - } + ..., + "streams": "ws://example.com:9985/api/v1/streams/valid_transactions", + ... } @@ -58,8 +56,8 @@ BigchainDB node will be ignored. Streams will always be under the WebSocket protocol (so ``ws://`` or ``wss://``) and accessible as extensions to the ``/api/v/streams/`` API root URL (for example, `validated transactions <#valid-transactions>`_ -would be accessible under ``/api/v1/streams/valid_tx``). If you're running your -own BigchainDB instance and need help determining its root URL, +would be accessible under ``/api/v1/streams/valid_transactions``). If you're +running your own BigchainDB instance and need help determining its root URL, then see the page titled :ref:`Determining the API Root URL`. All messages sent in a stream are in the JSON format. @@ -79,7 +77,7 @@ All messages sent in a stream are in the JSON format. Valid Transactions ~~~~~~~~~~~~~~~~~~ -``/valid_tx`` +``/valid_transactions`` Streams an event for any newly validated transactions. Message bodies contain the transaction's ID, associated asset ID, and containing block's ID. @@ -89,7 +87,7 @@ Example message: .. code:: JSON { - "tx_id": "", + "transaction_id": "", "asset_id": "", "block_id": "" } @@ -100,6 +98,6 @@ Example message: Transactions in BigchainDB are validated in batches ("blocks") and will, therefore, be streamed in batches. Each block can contain up to a 1000 transactions, ordered by the time at which they were included in the block. - The ``/valid_tx`` stream will send these transactions in the same order - that the block stored them in, but this does **NOT** guarantee that you - will recieve the events in that same order. + The ``/valid_transactions`` stream will send these transactions in the same + order that the block stored them in, but this does **NOT** guarantee that + you will recieve the events in that same order. diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 140ef50e..07e795a7 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -1,9 +1,3 @@ -############################################################### -# This config file runs bigchaindb:0.10.1 as a k8s Deployment # -# and it connects to the mongodb backend running as a # -# separate pod # -############################################################### - apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -18,13 +12,16 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:0.10.2 + image: bigchaindb/bigchaindb:1.0.0rc1 imagePullPolicy: IfNotPresent args: - start env: - name: BIGCHAINDB_DATABASE_HOST - value: mdb-instance-0 + valueFrom: + configMapKeyRef: + name: vars + key: mdb-instance-name - name: BIGCHAINDB_DATABASE_PORT value: "27017" - name: BIGCHAINDB_DATABASE_REPLICASET @@ -40,7 +37,10 @@ spec: - name: BIGCHAINDB_WSSERVER_PORT value: "9985" - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: "" + valueFrom: + configMapKeyRef: + name: bdb-public-key + key: bdb-public-key - name: BIGCHAINDB_KEYPAIR_PRIVATE value: "" - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY @@ -51,9 +51,22 @@ spec: value: "120" - name: BIGCHAINDB_LOG_LEVEL_CONSOLE value: debug + - name: BIGCHAINDB_DATABASE_CA_CERT + value: /etc/bigchaindb/ssl/ca.pem + - name: BIGCHAINDB_DATABASE_CRLFILE + value: /etc/bigchaindb/ssl/crlfile + - name: BIGCHAINDB_DATABASE_CERTFILE + value: /etc/bigchaindb/ssl/bdb-instance.pem + - name: BIGCHAINDB_DATABASE_KEYFILE + value: /etc/bigchaindb/ssl/bdb-instance.key + - name: BIGCHAINDB_DATABASE_LOGIN + value: /etc/bigchaindb/ssl/bdb-user # The following env var is not required for the bootstrap/first node #- name: BIGCHAINDB_KEYRING - # value: "" + # valueFrom: + # configMapKeyRef: + # name: bdb-keyring + # key: bdb-keyring ports: - containerPort: 9984 hostPort: 9984 @@ -63,6 +76,10 @@ spec: hostPort: 9985 name: bdb-ws-port protocol: TCP + volumeMounts: + - name: bdb-certs + mountPath: /etc/bigchaindb/ssl/ + readOnly: true resources: limits: cpu: 200m @@ -80,3 +97,8 @@ spec: initialDelaySeconds: 15 timeoutSeconds: 10 restartPolicy: Always + volumes: + - name: bdb-certs + secret: + secretName: bdb-certs + defaultMode: 0400 diff --git a/k8s/configuration/config-map.yaml b/k8s/configuration/config-map.yaml index 1c04dbf7..231f5b33 100644 --- a/k8s/configuration/config-map.yaml +++ b/k8s/configuration/config-map.yaml @@ -1,30 +1,22 @@ -####################################################### -# This YAML file desribes a ConfigMap for the cluster # -####################################################### +## Note: data values do NOT have to be base64-encoded in this file. +## vars is common environment variables for this BigchaindB node apiVersion: v1 kind: ConfigMap metadata: - name: mdb-mon + name: vars namespace: default data: - api-key: "" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mdb-backup - namespace: default -data: - api-key: "" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mdb-fqdn - namespace: default -data: - fqdn: mdb-instance-0 + # MongoDB + mdb-instance-name: "" + # BigchainDB + bdb-instance-name: "" + # NGINX + ngx-instance-name: "" + # MongoDB Monitoring Agent + mdb-mon-instance-name: "" + # MongoDB Backup Agent + mdb-bak-instance-name: "" --- apiVersion: v1 kind: ConfigMap @@ -32,5 +24,24 @@ metadata: name: mongodb-whitelist namespace: default data: + # We only support "all"" currently allowed-hosts: "all" - +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: bdb-keyring + namespace: default +data: + # Colon-separated list of all *other* nodes' BigchainDB public keys. + bdb-keyring: "<':' separated list of public keys>" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: bdb-public-key + namespace: default +data: + # BigchainDB public key of *this* node. + # Example: "EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce" + bdb-public-key: "" diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml new file mode 100644 index 00000000..f9c4aeb5 --- /dev/null +++ b/k8s/configuration/secret.yaml @@ -0,0 +1,119 @@ +# All secret data should be base64 encoded before embedding them here. +# Short strings can be encoded using, e.g. +# echo "secret string" | base64 -w 0 > secret.string.b64 +# Files (e.g. certificates) can be encoded using, e.g. +# cat cert.pem | base64 -w 0 > cert.pem.b64 +# then copy the contents of cert.pem.b64 (for example) below. +# Ref: https://kubernetes.io/docs/concepts/configuration/secret/ +# Unused values can be set to "" + +apiVersion: v1 +kind: Secret +metadata: + name: cloud-manager-credentials + namespace: default +type: Opaque +data: + # Base64-encoded Group ID + group-id: "" + # Base64-encoded Agent API Key + agent-api-key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: bdb-private-key + namespace: default +type: Opaque +data: + # Base64-encoded BigchainDB private key of *this* node + private.key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-instance.pem: "" + # Base64-encoded CA certificate (ca.crt) + ca.pem: "" + # Base64-encoded MongoDB CRL + mdb-crl.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-mon-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-mon-instance.pem: "" + # Base64-encoded CA certificate (ca.crt) + ca.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-bak-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-bak-instance.pem: "" + # Base64-encoded CA certificate (ca.crt) + ca.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: bdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded CA certificate (ca.crt) + ca.pem: "" + # Base64-encoded CRL file + crlfile: "" + # Base64-encoded BigchainDB instance certificate + bdb-instance.pem: "" + # Base64-encoded private key + bdb-instance.key: "" + # Base64-encoded instance authentication credentials + bdb-user: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-certs + namespace: default +type: Opaque +data: + # Base64-encoded HTTPS private key + cert.key: "" + # Base64-encoded HTTPS certificate chain + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert if from DigiCert, download "Best format for nginx". + cert.pem: "" + service-id: "" + version-header: "" + provider-key: "" + # The frontend-api-dns-name will be DNS name registered for your HTTPS + # certificate. + frontend-api-dns-name: "" + # The upstream-api-port can be set to any port other than 9984, 9985, 443, + # 8888 and 27017. We usually use port '9999', which is 'OTk5OQo=' in base 64. + upstream-api-port: "OTk5OQo=" diff --git a/k8s/logging-and-monitoring/analyze.py b/k8s/logging-and-monitoring/analyze.py new file mode 100644 index 00000000..eabb3438 --- /dev/null +++ b/k8s/logging-and-monitoring/analyze.py @@ -0,0 +1,77 @@ +""" +A little Python script to do some analysis of the NGINX logs. +To get the relevant NGINX logs: +1. Go to the OMS Portal +2. Create a new Log Search +3. Use a search string such as: + +Type=ContainerLog Image="bigchaindb/nginx_3scale:1.3" GET NOT("Go-http-client") NOT(runscope) + +(This gets all logs from the NGINX container, only those with the word "GET", +excluding those with the string "Go-http-client" [internal Kubernetes traffic], +excluding those with the string "runscope" [Runscope tests].) + +4. In the left sidebar, at the top, use the dropdown menu to select the time range, +e.g. "Data based on last 7 days". Pay attention to the number of results and +the time series chart in the left sidebar. Are there any spikes? +5. Export the search results. A CSV file will be saved on your local machine. +6. $ python3 analyze.py logs.csv + +Thanks to https://gist.github.com/hreeder/f1ffe1408d296ce0591d +""" + +import sys +import csv +import re +from dateutil.parser import parse + + +lineformat = re.compile(r'(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - ' + r'\[(?P\d{2}\/[a-z]{3}\/\d{4}:\d{2}:\d{2}:\d{2} ' + r'(\+|\-)\d{4})\] ((\"(GET|POST) )(?P.+)(http\/1\.1")) ' + r'(?P\d{3}) ' + r'(?P\d+) ' + r'(["](?P(\-)|(.+))["]) ' + r'(["](?P.+)["])', + re.IGNORECASE) + +filepath = sys.argv[1] + +logline_list = [] +with open(filepath) as csvfile: + csvreader = csv.reader(csvfile, delimiter=',') + for row in csvreader: + if row and (row[8] != 'LogEntry'): + # because the first line is just the column headers, such as 'LogEntry' + logline = row[8] + print(logline + '\n') + logline_data = re.search(lineformat, logline) + if logline_data: + logline_dict = logline_data.groupdict() + logline_list.append(logline_dict) + # so logline_list is a list of dicts + # print('{}'.format(logline_dict)) + +# Analysis + +total_bytes_sent = 0 +tstamp_list = [] + +for lldict in logline_list: + total_bytes_sent += int(lldict['bytessent']) + dt = lldict['dateandtime'] + # https://tinyurl.com/lqjnhot + dtime = parse(dt[:11] + " " + dt[12:]) + tstamp_list.append(dtime.timestamp()) + +print('Number of log lines seen: {}'.format(len(logline_list))) + +# Time range +trange_sec = max(tstamp_list) - min(tstamp_list) +trange_days = trange_sec / 60.0 / 60.0 / 24.0 +print('Time range seen (days): {}'.format(trange_days)) + +print('Total bytes sent: {}'.format(total_bytes_sent)) + +print('Average bytes sent per day (out via GET): {}'. + format(total_bytes_sent / trange_days)) diff --git a/k8s/mongodb-backup-agent/container/Dockerfile b/k8s/mongodb-backup-agent/container/Dockerfile index 8407fb09..e70ee3d3 100644 --- a/k8s/mongodb-backup-agent/container/Dockerfile +++ b/k8s/mongodb-backup-agent/container/Dockerfile @@ -6,7 +6,10 @@ ARG FILE_URL="https://cloud.mongodb.com/download/agent/backup/"$DEB_FILE WORKDIR / RUN apt update \ && apt -y upgrade \ - && apt -y install --no-install-recommends curl ca-certificates logrotate \ + && apt -y install --no-install-recommends \ + curl \ + ca-certificates \ + logrotate \ libsasl2-2 \ && curl -OL $FILE_URL \ && dpkg -i $DEB_FILE \ @@ -16,4 +19,6 @@ RUN apt update \ && apt clean COPY mongodb_backup_agent_entrypoint.bash / RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/ +VOLUME /etc/mongod/ssl +#USER mongodb-mms-agent - BUG(Krish) Uncomment after tests are complete ENTRYPOINT ["/mongodb_backup_agent_entrypoint.bash"] diff --git a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash index 5d1780ea..770dc2b7 100755 --- a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash +++ b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/mongodb-backup-agent:2.0 . +docker build -t bigchaindb/mongodb-backup-agent:3.0 . -docker push bigchaindb/mongodb-backup-agent:2.0 +docker push bigchaindb/mongodb-backup-agent:3.0 diff --git a/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash b/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash index 6b663fe9..13a40bb5 100755 --- a/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash +++ b/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash @@ -4,21 +4,29 @@ set -euo pipefail MONGODB_BACKUP_CONF_FILE=/etc/mongodb-mms/backup-agent.config -mms_api_key=`printenv MMS_API_KEY` +mms_api_keyfile_path=`printenv MMS_API_KEYFILE_PATH` +mms_groupid_keyfile_path=`printenv MMS_GROUPID_KEYFILE_PATH` ca_crt_path=`printenv CA_CRT_PATH` backup_crt_path=`printenv BACKUP_PEM_PATH` -if [[ -z "${mms_api_key}" || \ +if [[ -z "${mms_api_keyfile_path}" || \ -z "${ca_crt_path}" || \ - -z "${backup_crt_path}" ]]; then + -z "${backup_crt_path}" || \ + -z "${mms_groupid_keyfile_path}" ]]; then echo "Invalid environment settings detected. Exiting!" exit 1 fi sed -i '/mmsApiKey/d' ${MONGODB_BACKUP_CONF_FILE} +sed -i '/mmsGroupId/d' ${MONGODB_BACKUP_CONF_FILE} sed -i '/mothership/d' ${MONGODB_BACKUP_CONF_FILE} +# Get the api key from file +mms_api_key=`cat ${mms_api_keyfile_path}` +mms_groupid_key=`cat ${mms_groupid_keyfile_path}` + echo "mmsApiKey="${mms_api_key} >> ${MONGODB_BACKUP_CONF_FILE} +echo "mmsGroupId="${mms_groupid_key} >> ${MONGODB_BACKUP_CONF_FILE} echo "mothership=api-backup.eu-west-1.mongodb.com" >> ${MONGODB_BACKUP_CONF_FILE} # Append SSL settings to the config file diff --git a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml index b3d5a9ec..74f89247 100644 --- a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml +++ b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml @@ -1,27 +1,58 @@ +############################################################ +# This config file defines a k8s Deployment for the # +# bigchaindb/mongodb-backup-agent Docker image # +# # +# It connects to a MongoDB instance in a separate pod, # +# all remote MongoDB instances in the cluster, # +# and also to MongoDB Cloud Manager (an external service). # +# Notes: # +# MongoDB agents connect to Cloud Manager on port 443. # +############################################################ + apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: mdb-backup-instance-0-dep + name: mdb-bak-instance-0-dep spec: replicas: 1 template: metadata: + name: mdb-bak-instance-0-dep labels: - app: mdb-backup-instance-0-dep + app: mdb-bak-instance-0-dep spec: terminationGracePeriodSeconds: 10 containers: - name: mdb-backup - image: bigchaindb/mongodb-backup-agent:1.0 + image: bigchaindb/mongodb-backup-agent:3.0 imagePullPolicy: Always env: - - name: MMS_API_KEY - valueFrom: - configMapKeyRef: - name: mdb-backup - key: api-key + - name: MMS_API_KEYFILE_PATH + value: /etc/mongod/cloud/agent-api-key + - name: MMS_GROUPID_KEYFILE_PATH + value: /etc/mongod/cloud/group-id + - name: CA_CRT_PATH + value: /etc/mongod/ssl/ca.pem + - name: BACKUP_PEM_PATH + value: /etc/mongod/ssl/mdb-bak-instance.pem resources: limits: cpu: 200m memory: 768Mi + volumeMounts: + - name: mdb-bak-certs + mountPath: /etc/mongod/ssl/ + readOnly: true + - name: cloud-manager-credentials + mountPath: /etc/mongod/cloud/ + readOnly: true restartPolicy: Always + volumes: + - name: mdb-bak-certs + secret: + secretName: mdb-bak-certs + defaultMode: 0400 + - name: cloud-manager-credentials + secret: + secretName: cloud-manager-credentials + defaultMode: 0400 diff --git a/k8s/mongodb-monitoring-agent/container/Dockerfile b/k8s/mongodb-monitoring-agent/container/Dockerfile index ec6496d8..d6a16ed0 100644 --- a/k8s/mongodb-monitoring-agent/container/Dockerfile +++ b/k8s/mongodb-monitoring-agent/container/Dockerfile @@ -18,7 +18,10 @@ ARG FILE_URL="https://cloud.mongodb.com/download/agent/monitoring/"$DEB_FILE WORKDIR / RUN apt update \ && apt -y upgrade \ - && apt -y install --no-install-recommends curl ca-certificates logrotate \ + && apt -y install --no-install-recommends \ + curl \ + ca-certificates \ + logrotate \ libsasl2-2 \ && curl -OL $FILE_URL \ && dpkg -i $DEB_FILE \ @@ -50,5 +53,6 @@ RUN apt update \ COPY mongodb_mon_agent_entrypoint.bash / RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/ +VOLUME /etc/mongod/ssl #USER mongodb-mms-agent - BUG(Krish) Uncomment after tests are complete ENTRYPOINT ["/mongodb_mon_agent_entrypoint.bash"] diff --git a/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash b/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash index caefb6d7..2bd5aeb5 100755 --- a/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash +++ b/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/mongodb-monitoring-agent:2.0 . +docker build -t bigchaindb/mongodb-monitoring-agent:3.0 . -docker push bigchaindb/mongodb-monitoring-agent:2.0 +docker push bigchaindb/mongodb-monitoring-agent:3.0 diff --git a/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash b/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash index 9ef96303..7ae161e3 100755 --- a/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash +++ b/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash @@ -8,24 +8,33 @@ set -euo pipefail MONGODB_MON_CONF_FILE=/etc/mongodb-mms/monitoring-agent.config -mms_api_key=`printenv MMS_API_KEY` +mms_api_keyfile_path=`printenv MMS_API_KEYFILE_PATH` +mms_groupid_keyfile_path=`printenv MMS_GROUPID_KEYFILE_PATH` ca_crt_path=`printenv CA_CRT_PATH` monitoring_crt_path=`printenv MONITORING_PEM_PATH` -if [[ -z "${mms_api_key}" || \ +if [[ -z "${mms_api_keyfile_path}" || \ -z "${ca_crt_path}" || \ - -z "${monitoring_crt_path}" ]]; then + -z "${monitoring_crt_path}" || \ + -z "${mms_groupid_keyfile_path}" ]]; then echo "Invalid environment settings detected. Exiting!" exit 1 fi -# Delete all lines containing "mmsApiKey" in the MongoDB Monitoring Agent -# config file /etc/mongodb-mms/monitoring-agent.config +# Delete the line containing "mmsApiKey" and the line containing "mmsGroupId" +# in the MongoDB Monitoring Agent config file +# /etc/mongodb-mms/monitoring-agent.config sed -i '/mmsApiKey/d' $MONGODB_MON_CONF_FILE +sed -i '/mmsGroupId/d' $MONGODB_MON_CONF_FILE + +# Get the api key from file +mms_api_key=`cat ${mms_api_keyfile_path}` +mms_groupid_key=`cat ${mms_groupid_keyfile_path}` # Append a new line of the form # mmsApiKey=value_of_MMS_API_KEY echo "mmsApiKey="${mms_api_key} >> ${MONGODB_MON_CONF_FILE} +echo "mmsGroupId="${mms_groupid_key} >> ${MONGODB_MON_CONF_FILE} # Append SSL settings to the config file echo "useSslForAllConnections=true" >> ${MONGODB_MON_CONF_FILE} diff --git a/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml b/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml index 98abe92b..4ddb233d 100644 --- a/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml +++ b/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml @@ -1,6 +1,6 @@ ############################################################ # This config file defines a k8s Deployment for the # -# bigchaindb/mongodb-monitoring-agent:latest Docker image # +# bigchaindb/mongodb-monitoring-agent Docker image # # # # It connects to a MongoDB instance in a separate pod, # # all remote MongoDB instances in the cluster, # @@ -17,22 +17,42 @@ spec: replicas: 1 template: metadata: + name: mdb-mon-instance-0-dep labels: app: mdb-mon-instance-0-dep spec: terminationGracePeriodSeconds: 10 containers: - name: mdb-mon - image: bigchaindb/mongodb-monitoring-agent:1.0 + image: bigchaindb/mongodb-monitoring-agent:3.0 imagePullPolicy: Always env: - - name: MMS_API_KEY - valueFrom: - configMapKeyRef: - name: mdb-mon - key: api-key + - name: MMS_API_KEYFILE_PATH + value: /etc/mongod/cloud/agent-api-key + - name: MMS_GROUPID_KEYFILE_PATH + value: /etc/mongod/cloud/group-id + - name: CA_CRT_PATH + value: /etc/mongod/ssl/ca.pem + - name: MONITORING_PEM_PATH + value: /etc/mongod/ssl/mdb-mon-instance.pem resources: limits: cpu: 200m memory: 768Mi + volumeMounts: + - name: mdb-mon-certs + mountPath: /etc/mongod/ssl/ + readOnly: true + - name: cloud-manager-credentials + mountPath: /etc/mongod/cloud/ + readOnly: true restartPolicy: Always + volumes: + - name: mdb-mon-certs + secret: + secretName: mdb-mon-certs + defaultMode: 0400 + - name: cloud-manager-credentials + secret: + secretName: cloud-manager-credentials + defaultMode: 0400 diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile index 66b076c7..58a7f88f 100644 --- a/k8s/mongodb/container/Dockerfile +++ b/k8s/mongodb/container/Dockerfile @@ -4,10 +4,9 @@ WORKDIR / RUN apt-get update \ && apt-get -y upgrade \ && apt-get autoremove \ - && apt-get clean \ - && mkdir /mongo-ssl + && apt-get clean COPY mongod.conf.template /etc/mongod.conf COPY mongod_entrypoint.bash / -VOLUME /data/db /data/configdb /mongo-ssl +VOLUME /data/db /data/configdb /etc/mongod/ssl EXPOSE 27017 ENTRYPOINT ["/mongod_entrypoint.bash"] diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md index 9f9c46d1..4cec6250 100644 --- a/k8s/mongodb/container/README.md +++ b/k8s/mongodb/container/README.md @@ -9,9 +9,11 @@ * We also need a way to overwrite certain parameters to suit our use case. -### Step 1: Build the Latest Container - -`docker build -t bigchaindb/mongodb:3.4.4 .` from the root of this project. +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. ### Step 2: Run the Container @@ -25,7 +27,7 @@ docker run \ --volume=:/data/db \ --volume=:/data/configdb \ --volume=:/mongo-ssl:ro \ - bigchaindb/mongodb:3.4.4 \ + bigchaindb/mongodb:3.0 \ --mongodb-port \ --mongodb-key-file-path /mongo-ssl/.pem \ --mongodb-key-file-password \ diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash new file mode 100755 index 00000000..44806682 --- /dev/null +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t bigchaindb/mongodb:3.0 . + +docker push bigchaindb/mongodb:3.0 diff --git a/k8s/mongodb/container/mongod.conf.template b/k8s/mongodb/container/mongod.conf.template index 5b5f5c1f..089313d5 100644 --- a/k8s/mongodb/container/mongod.conf.template +++ b/k8s/mongodb/container/mongod.conf.template @@ -65,14 +65,15 @@ net: #weakCertificateValidation: false #allowInvalidCertificates: false -#security: TODO -# authorization: enabled -# clusterAuthMode: x509 +security: + authorization: enabled + clusterAuthMode: x509 setParameter: enableLocalhostAuthBypass: true - #notablescan: 1 TODO - #logUserIds: 1 TODO + #notablescan: 1 + logUserIds: 1 + authenticationMechanisms: MONGODB-X509,SCRAM-SHA-1 storage: dbPath: /data/db diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index 2f180929..c12d5f39 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -21,23 +21,37 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: mongodb - image: bigchaindb/mongodb:3.4.3 - imagePullPolicy: IfNotPresent + image: bigchaindb/mongodb:3.4.4 + imagePullPolicy: Always env: - name: MONGODB_FQDN valueFrom: configMapKeyRef: - name: mdb-fqdn - key: fqdn + name: vars + key: mdb-instance-name - name: MONGODB_POD_IP valueFrom: fieldRef: fieldPath: status.podIP + - name: MONGODB_REPLICA_SET_NAME + value: bigchain-rs + - name: MONGODB_PORT + value: "27017" args: - - --replica-set-name=bigchain-rs - - --fqdn=$(MONGODB_FQDN) - - --port=27017 - - --ip=$(MONGODB_POD_IP) + - --mongodb-port + - $(MONGODB_PORT) + - --mongodb-key-file-path + - /etc/mongod/ssl/mdb-instance.pem + - --mongodb-ca-file-path + - /etc/mongod/ssl/ca.pem + - --mongodb-crl-file-path + - /etc/mongod/ssl/mdb-crl.pem + - --replica-set-name + - $(MONGODB_REPLICA_SET_NAME) + - --mongodb-fqdn + - $(MONGODB_FQDN) + - --mongodb-ip + - $(MONGODB_POD_IP) securityContext: capabilities: add: @@ -52,6 +66,9 @@ spec: mountPath: /data/db - name: mdb-configdb mountPath: /data/configdb + - name: mdb-certs + mountPath: /etc/mongod/ssl/ + readOnly: true resources: limits: cpu: 200m @@ -71,3 +88,7 @@ spec: - name: mdb-configdb persistentVolumeClaim: claimName: mongo-configdb-claim + - name: mdb-certs + secret: + secretName: mdb-certs + defaultMode: 0400 diff --git a/k8s/nginx-3scale/nginx-3scale-dep.yaml b/k8s/nginx-3scale/nginx-3scale-dep.yaml index 964cbf8b..7951e14d 100644 --- a/k8s/nginx-3scale/nginx-3scale-dep.yaml +++ b/k8s/nginx-3scale/nginx-3scale-dep.yaml @@ -19,8 +19,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: nginx-3scale - image: bigchaindb/nginx_3scale:1.1 - # TODO(Krish): Change later to IfNotPresent + image: bigchaindb/nginx_3scale:1.5 imagePullPolicy: Always env: - name: MONGODB_FRONTEND_PORT @@ -33,7 +32,6 @@ spec: - name: BIGCHAINDB_FRONTEND_PORT value: $(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT) - name: BIGCHAINDB_BACKEND_HOST - # NGINX requires FQDN to resolve names value: bdb-instance-0.default.svc.cluster.local - name: BIGCHAINDB_BACKEND_PORT value: "9984" @@ -46,19 +44,6 @@ spec: value: "10.0.0.10" - name: NGINX_HEALTH_CHECK_PORT value: "8888" - # TODO(Krish): use secrets for sensitive info - - name: THREESCALE_SECRET_TOKEN - value: "" - - name: THREESCALE_SERVICE_ID - value: "" - - name: THREESCALE_VERSION_HEADER - value: "" - - name: THREESCALE_PROVIDER_KEY - value: "" - - name: THREESCALE_FRONTEND_API_DNS_NAME - value: "" - - name: THREESCALE_UPSTREAM_API_PORT - value: "" ports: - containerPort: 27017 hostPort: 27017 @@ -81,7 +66,10 @@ spec: name: public-api-port protocol: TCP volumeMounts: - - name: https + - name: threescale-credentials + mountPath: /usr/local/openresty/nginx/conf/threescale + readOnly: true + - name: https-certs mountPath: /usr/local/openresty/nginx/conf/ssl/ readOnly: true resources: @@ -96,7 +84,11 @@ spec: timeoutSeconds: 10 restartPolicy: Always volumes: - - name: https + - name: https-certs secret: - secretName: certs + secretName: https-certs + defaultMode: 0400 + - name: threescale-credentials + secret: + secretName: threescale-credentials defaultMode: 0400 diff --git a/k8s/nginx-3scale/nginx-3scale-secret.yaml b/k8s/nginx-3scale/nginx-3scale-secret.yaml deleted file mode 100644 index 8f725313..00000000 --- a/k8s/nginx-3scale/nginx-3scale-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Certificate data should be base64 encoded before embedding them here by using -# `cat cert.pem | base64 -w 0 > cert.pem.b64` and then copy the resulting -# value here. Same goes for cert.key. -# Ref: https://kubernetes.io/docs/concepts/configuration/secret/ - -apiVersion: v1 -kind: Secret -metadata: - name: certs -type: Opaque -data: - cert.pem: - cert.key: diff --git a/k8s/nginx/container/README.md b/k8s/nginx/container/README.md index 5a1e1273..f8baaba2 100644 --- a/k8s/nginx/container/README.md +++ b/k8s/nginx/container/README.md @@ -22,10 +22,10 @@ ### Step 1: Build the Latest Container -Run `docker build -t bigchaindb/nginx: .` from this folder. +Run `docker build -t bigchaindb/nginx:1.0 .` from this folder. Optional: Upload container to Docker Hub: -`docker push bigchaindb/nginx:` +`docker push bigchaindb/nginx:1.0` ### Step 2: Run the Container @@ -34,41 +34,41 @@ format, eg: `1.2.3.4/16` ``` docker run \ ---env "MONGODB_FRONTEND_PORT=" \ ---env "MONGODB_BACKEND_HOST=" \ ---env "MONGODB_BACKEND_PORT=" \ ---env "BIGCHAINDB_FRONTEND_PORT=" \ ---env "BIGCHAINDB_BACKEND_HOST=" \ ---env "BIGCHAINDB_BACKEND_PORT=" \ ---env "BIGCHAINDB_WS_BACKEND_PORT=" \ ---env "BIGCHAINDB_WS_FRONTEND_PORT=" \ ---env "MONGODB_WHITELIST=" \ ---env "DNS_SERVER=" \ ---name=ngx \ ---publish=: \ ---publish=: \ ---rm=true \ -bigchaindb/nginx + --env "MONGODB_FRONTEND_PORT=" \ + --env "MONGODB_BACKEND_HOST=" \ + --env "MONGODB_BACKEND_PORT=" \ + --env "BIGCHAINDB_FRONTEND_PORT=" \ + --env "BIGCHAINDB_BACKEND_HOST=" \ + --env "BIGCHAINDB_BACKEND_PORT=" \ + --env "BIGCHAINDB_WS_BACKEND_PORT=" \ + --env "BIGCHAINDB_WS_FRONTEND_PORT=" \ + --env "MONGODB_WHITELIST=" \ + --env "DNS_SERVER=" \ + --name=ngx \ + --publish=: \ + --publish=: \ + --rm=true \ + bigchaindb/nginx:1.0 ``` For example: ``` docker run \ ---env "MONGODB_FRONTEND_PORT=17017" \ ---env "MONGODB_BACKEND_HOST=localhost" \ ---env "MONGODB_BACKEND_PORT=27017" \ ---env "BIGCHAINDB_FRONTEND_PORT=80" \ ---env "BIGCHAINDB_BACKEND_HOST=localhost" \ ---env "BIGCHAINDB_BACKEND_PORT=9984" \ ---env="BIGCHAINDB_WS_FRONTEND_PORT=81" \ ---env="BIGCHAINDB_WS_BACKEND_PORT=9985" \ ---env "MONGODB_WHITELIST=192.168.0.0/16:10.0.2.0/24" \ ---name=ngx \ ---publish=80:80 \ ---publish=17017:17017 \ ---rm=true \ -bigchaindb/nginx + --env="MONGODB_FRONTEND_PORT=17017" \ + --env="MONGODB_BACKEND_HOST=localhost" \ + --env="MONGODB_BACKEND_PORT=27017" \ + --env="BIGCHAINDB_FRONTEND_PORT=80" \ + --env="BIGCHAINDB_BACKEND_HOST=localhost" \ + --env="BIGCHAINDB_BACKEND_PORT=9984" \ + --env="BIGCHAINDB_WS_FRONTEND_PORT=81" \ + --env="BIGCHAINDB_WS_BACKEND_PORT=9985" \ + --env="MONGODB_WHITELIST=192.168.0.0/16:10.0.2.0/24" \ + --env="DNS_SERVER=127.0.0.1" \ + --name=ngx \ + --publish=80:80 \ + --publish=17017:17017 \ + --rm=true \ + bigchaindb/nginx:1.0 ``` ### Note: diff --git a/k8s/nginx/nginx-dep.yaml b/k8s/nginx/nginx-dep.yaml index 0aad0b2d..133e2688 100644 --- a/k8s/nginx/nginx-dep.yaml +++ b/k8s/nginx/nginx-dep.yaml @@ -20,22 +20,24 @@ spec: containers: - name: nginx image: bigchaindb/nginx:1.0 - imagePullPolicy: IfNotPresent + imagePullPolicy: Always env: - name: MONGODB_FRONTEND_PORT - value: $(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_MDB_PORT) + value: "27017" - name: MONGODB_BACKEND_HOST - # NGINX requires FQDN to resolve names value: mdb-instance-0.default.svc.cluster.local - name: MONGODB_BACKEND_PORT value: "27017" - name: BIGCHAINDB_FRONTEND_PORT - value: $(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT) + value: "80" - name: BIGCHAINDB_BACKEND_HOST - # NGINX requires FQDN to resolve names value: bdb-instance-0.default.svc.cluster.local - name: BIGCHAINDB_BACKEND_PORT value: "9984" + - name: BIGCHAINDB_WS_FRONTEND_PORT + value: "81" + - name: BIGCHAINDB_WS_BACKEND_PORT + value: "9985" - name: DNS_SERVER value: "10.0.0.10" - name: MONGODB_WHITELIST @@ -43,10 +45,6 @@ spec: configMapKeyRef: name: mongodb-whitelist key: allowed-hosts - - name: BIGCHAINDB_WS_FRONTEND_PORT - value: "81" - - name: BIGCHAINDB_WS_BACKEND_PORT - value: "9985" ports: - containerPort: 27017 hostPort: 27017 diff --git a/k8s/nginx/nginx-svc.yaml b/k8s/nginx/nginx-svc.yaml index b9d8bcaf..1e1e131c 100644 --- a/k8s/nginx/nginx-svc.yaml +++ b/k8s/nginx/nginx-svc.yaml @@ -19,7 +19,7 @@ spec: protocol: TCP - port: 80 targetPort: 80 - name: ngx-public-bdb-port + name: ngx-public-api-port protocol: TCP - port: 81 targetPort: 81 diff --git a/rdb.yml b/rdb.yml new file mode 100644 index 00000000..15f91675 --- /dev/null +++ b/rdb.yml @@ -0,0 +1,48 @@ +version: '2' + +services: + rdb: + image: rethinkdb + ports: + - "58080:8080" + - "28015" + volumes_from: + - rdb-data + + rdb-2: + image: rethinkdb + ports: + - "8080" + - "29015" + command: rethinkdb --join rdb:29015 --bind all + + rdb-data: + image: rethinkdb:2.3.5 + volumes: + - /data + command: "true" + + bdb-rdb: + build: + context: . + dockerfile: Dockerfile-dev + args: + backend: rethinkdb + container_name: docker-bigchaindb + volumes: + - ./bigchaindb:/usr/src/app/bigchaindb + - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./k8s:/usr/src/app/k8s + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini + - ./Makefile:/usr/src/app/Makefile + environment: + BIGCHAINDB_DATABASE_BACKEND: rethinkdb + BIGCHAINDB_DATABASE_HOST: rdb + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + ports: + - "9984" + command: bigchaindb start diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md new file mode 100644 index 00000000..befe2400 --- /dev/null +++ b/scripts/benchmarks/README.md @@ -0,0 +1,40 @@ +# Benchmarks + +## CREATE transaction throughput + +This is a measurement of the throughput of CREATE transactions through the entire +pipeline, ie, the web frontend, block creation, and block validation, where the +output of the measurement is transactions per second. + +The benchmark runs for a fixed period of time and makes metrics available via +a graphite interface. + +### Running the benchmark + +Dependencies: + +* Python 3.5+ +* docker-compose 1.8.0+ +* docker 1.12+ + +To start: + + $ python3 scripts/benchmarks/create_thoughtput.py + +To start using a separate namespace for docker-compose: + + $ COMPOSE_PROJECT_NAME=somename python3 scripts/benchmarks/create_thoughtput.py + +### Results + +A test was run on AWS with the following instance configuration: + +* Ubuntu Server 16.04 (ami-060cde69) +* 32 core compute optimized (c3.8xlarge) +* 100gb root volume (300/3000 IOPS) + +The server received and validated over 800 transactions per second: + +![BigchainDB transaction throughput](https://cloud.githubusercontent.com/assets/125019/26688641/85d56d1e-46f3-11e7-8148-bf3bc8c54c33.png) + +For more information on how the benchmark was run, the abridged session buffer [is available](https://gist.github.com/libscott/8a37c5e134b2d55cfb55082b1cd85a02). diff --git a/scripts/benchmarks/create_thoughtput.py b/scripts/benchmarks/create_thoughtput.py new file mode 100644 index 00000000..2fe8c557 --- /dev/null +++ b/scripts/benchmarks/create_thoughtput.py @@ -0,0 +1,133 @@ +import sys +import math +import time +import requests +import subprocess +import multiprocessing + + +def main(): + cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d mdb') + cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d bdb') + cmd('docker-compose -f docker-compose.yml -f benchmark.yml up -d graphite') + + out = cmd('docker-compose -f benchmark.yml port graphite 80', capture=True) + graphite_web = 'http://localhost:%s/' % out.strip().split(':')[1] + print('Graphite web interface at: ' + graphite_web) + + start = time.time() + + cmd('docker-compose -f docker-compose.yml -f benchmark.yml exec bdb python %s load' % sys.argv[0]) + + mins = math.ceil((time.time() - start) / 60) + 1 + + graph_url = graphite_web + 'render/?width=900&height=600&_salt=1495462891.335&target=stats.pipelines.block.throughput&target=stats.pipelines.vote.throughput&target=stats.web.tx.post&from=-%sminutes' % mins # noqa + + print(graph_url) + + +def load(): + from bigchaindb.core import Bigchain + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.common.transaction import Transaction + + def transactions(): + priv, pub = generate_key_pair() + tx = Transaction.create([pub], [([pub], 1)]) + while True: + i = yield tx.to_dict() + tx.asset = {'data': {'n': i}} + tx.sign([priv]) + + def wait_for_up(): + print('Waiting for server to start... ', end='') + while True: + try: + requests.get('http://localhost:9984/') + break + except requests.ConnectionError: + time.sleep(0.1) + print('Ok') + + def post_txs(): + txs = transactions() + txs.send(None) + try: + with requests.Session() as session: + while True: + i = tx_queue.get() + if i is None: + break + tx = txs.send(i) + res = session.post('http://localhost:9984/api/v1/transactions/', json=tx) + assert res.status_code == 202 + except KeyboardInterrupt: + pass + + wait_for_up() + num_clients = 30 + test_time = 60 + tx_queue = multiprocessing.Queue(maxsize=num_clients) + txn = 0 + b = Bigchain() + + start_time = time.time() + + for i in range(num_clients): + multiprocessing.Process(target=post_txs).start() + + print('Sending transactions') + while time.time() - start_time < test_time: + # Post 500 transactions to the server + for i in range(500): + tx_queue.put(txn) + txn += 1 + print(txn) + while True: + # Wait for the server to reduce the backlog to below + # 10000 transactions. The expectation is that 10000 transactions + # will not be processed faster than a further 500 transactions can + # be posted, but nonetheless will be processed within a few seconds. + # This keeps the test from running on and keeps the transactions from + # being considered stale. + count = b.connection.db.backlog.count() + if count > 10000: + time.sleep(0.2) + else: + break + + for i in range(num_clients): + tx_queue.put(None) + + print('Waiting to clear backlog') + while True: + bl = b.connection.db.backlog.count() + if bl == 0: + break + print(bl) + time.sleep(1) + + print('Waiting for all votes to come in') + while True: + blocks = b.connection.db.bigchain.count() + votes = b.connection.db.votes.count() + if blocks == votes + 1: + break + print('%s blocks, %s votes' % (blocks, votes)) + time.sleep(3) + + print('Finished') + + +def cmd(command, capture=False): + stdout = subprocess.PIPE if capture else None + args = ['bash', '-c', command] + proc = subprocess.Popen(args, stdout=stdout) + assert not proc.wait() + return capture and proc.stdout.read().decode() + + +if sys.argv[1:] == ['load']: + load() +else: + main() diff --git a/setup.py b/setup.py index 4fd485c0..e2db673c 100644 --- a/setup.py +++ b/setup.py @@ -67,9 +67,9 @@ install_requires = [ 'rethinkdb~=2.3', # i.e. a version between 2.3 and 3.0 'pymongo~=3.4', 'pysha3~=1.0.2', - 'cryptoconditions>=0.5.0', + 'cryptoconditions~=0.6.0.dev', 'python-rapidjson==0.0.11', - 'logstats>=0.2.1', + 'logstats~=0.2.1', 'flask>=0.10.1', 'flask-cors~=3.0.0', 'flask-restful~=0.3.0', @@ -80,6 +80,7 @@ install_requires = [ 'pyyaml~=3.12', 'aiohttp~=2.0', 'python-rapidjson-schema==0.1.1', + 'statsd==3.2.1', ] setup( @@ -111,8 +112,8 @@ setup( 'Natural Language :: English', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX :: Linux', ], @@ -125,7 +126,7 @@ setup( ], }, install_requires=install_requires, - setup_requires=['pytest-runner'], + setup_requires=['pytest-runner', 'cryptoconditions'], tests_require=tests_require, extras_require={ 'test': tests_require, diff --git a/tests/README.md b/tests/README.md index e6de82c9..252fcda4 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,6 +28,13 @@ instructions for how to do that. Next, make sure you have RethinkDB or MongoDB running in the background. You can run RethinkDB using `rethinkdb --daemon` or MongoDB using `mongod --replSet=bigchain-rs`. +If you wish to test with a TLS/SSL enabled MongoDB, use the command +```text +mongod --replSet=bigchain-rs --sslAllowInvalidHostnames --sslMode=requireSSL \ +-sslCAFile=bigchaindb/tests/backend/mongodb-ssl/certs/ca.crt \ +--sslCRLFile=bigchaindb/tests/backend/mongodb-ssl/certs/crl.pem \ +--sslPEMKeyFile=bigchaindb/tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem +``` The `pytest` command has many options. If you want to learn about all the things you can do with pytest, see [the pytest @@ -56,7 +63,12 @@ python setup.py test **Note**: the above pytest commands default to use RethinkDB as the backend. If you wish to run the tests against MongoDB add the `--database-backend=mongodb` -to the `pytest` command. +to the `pytest` command. If you wish to run tests against a TLS/SSL enabled +MongoDB instance (as mentioned above), use the command +```text +pytest -v --database-backend=mongodb-ssl -m bdb_ssl +``` + How does `python setup.py test` work? The documentation for [pytest-runner](https://pypi.python.org/pypi/pytest-runner) explains. diff --git a/tests/assets/test_divisible_assets.py b/tests/assets/test_divisible_assets.py index e1ea726f..123fe063 100644 --- a/tests/assets/test_divisible_assets.py +++ b/tests/assets/test_divisible_assets.py @@ -52,8 +52,8 @@ def test_single_in_single_own_single_out_multiple_own_create(b, user_pk): assert tx_signed.outputs[0].amount == 100 output = tx_signed.outputs[0].to_dict() - assert 'subfulfillments' in output['condition']['details'] - assert len(output['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in output['condition']['details'] + assert len(output['condition']['details']['subconditions']) == 2 assert len(tx_signed.inputs) == 1 @@ -76,8 +76,8 @@ def test_single_in_single_own_multiple_out_mix_own_create(b, user_pk): assert tx_signed.outputs[1].amount == 50 output_cid1 = tx_signed.outputs[1].to_dict() - assert 'subfulfillments' in output_cid1['condition']['details'] - assert len(output_cid1['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in output_cid1['condition']['details'] + assert len(output_cid1['condition']['details']['subconditions']) == 2 assert len(tx_signed.inputs) == 1 @@ -89,6 +89,7 @@ def test_single_in_single_own_multiple_out_mix_own_create(b, user_pk): def test_single_in_multiple_own_single_out_single_own_create(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details tx = Transaction.create([b.me, user_pk], [([user_pk], 100)]) tx_signed = tx.sign([b.me_private, user_sk]) @@ -97,9 +98,9 @@ def test_single_in_multiple_own_single_out_single_own_create(b, user_pk, assert tx_signed.outputs[0].amount == 100 assert len(tx_signed.inputs) == 1 - ffill = tx_signed.inputs[0].fulfillment.to_dict() - assert 'subfulfillments' in ffill - assert len(ffill['subfulfillments']) == 2 + ffill = _fulfillment_to_details(tx_signed.inputs[0].fulfillment) + assert 'subconditions' in ffill + assert len(ffill['subconditions']) == 2 # TRANSFER divisible asset @@ -207,8 +208,8 @@ def test_single_in_single_own_single_out_multiple_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 condition = tx_transfer_signed.outputs[0].to_dict() - assert 'subfulfillments' in condition['condition']['details'] - assert len(condition['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in condition['condition']['details'] + assert len(condition['condition']['details']['subconditions']) == 2 assert len(tx_transfer_signed.inputs) == 1 @@ -248,8 +249,8 @@ def test_single_in_single_own_multiple_out_mix_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[1].amount == 50 output_cid1 = tx_transfer_signed.outputs[1].to_dict() - assert 'subfulfillments' in output_cid1['condition']['details'] - assert len(output_cid1['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' in output_cid1['condition']['details'] + assert len(output_cid1['condition']['details']['subconditions']) == 2 assert len(tx_transfer_signed.inputs) == 1 @@ -264,6 +265,7 @@ def test_single_in_single_own_multiple_out_mix_own_transfer(b, user_pk, def test_single_in_multiple_own_single_out_single_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([b.me, user_pk], 100)]) @@ -286,9 +288,9 @@ def test_single_in_multiple_own_single_out_single_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 assert len(tx_transfer_signed.inputs) == 1 - ffill = tx_transfer_signed.inputs[0].fulfillment.to_dict() - assert 'subfulfillments' in ffill - assert len(ffill['subfulfillments']) == 2 + ffill = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + assert 'subconditions' in ffill + assert len(ffill['subconditions']) == 2 # TRANSFER divisible asset @@ -334,6 +336,7 @@ def test_multiple_in_single_own_single_out_single_own_transfer(b, user_pk, def test_multiple_in_multiple_own_single_out_single_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([user_pk, b.me], 50), ([user_pk, b.me], 50)]) @@ -356,12 +359,12 @@ def test_multiple_in_multiple_own_single_out_single_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 assert len(tx_transfer_signed.inputs) == 2 - ffill_fid0 = tx_transfer_signed.inputs[0].fulfillment.to_dict() - ffill_fid1 = tx_transfer_signed.inputs[1].fulfillment.to_dict() - assert 'subfulfillments' in ffill_fid0 - assert 'subfulfillments' in ffill_fid1 - assert len(ffill_fid0['subfulfillments']) == 2 - assert len(ffill_fid1['subfulfillments']) == 2 + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid0['subconditions']) == 2 + assert len(ffill_fid1['subconditions']) == 2 # TRANSFER divisible asset @@ -375,6 +378,7 @@ def test_multiple_in_multiple_own_single_out_single_own_transfer(b, user_pk, def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([user_pk], 50), ([user_pk, b.me], 50)]) @@ -397,11 +401,11 @@ def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(b, user_pk, assert tx_transfer_signed.outputs[0].amount == 100 assert len(tx_transfer_signed.inputs) == 2 - ffill_fid0 = tx_transfer_signed.inputs[0].fulfillment.to_dict() - ffill_fid1 = tx_transfer_signed.inputs[1].fulfillment.to_dict() - assert 'subfulfillments' not in ffill_fid0 - assert 'subfulfillments' in ffill_fid1 - assert len(ffill_fid1['subfulfillments']) == 2 + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' not in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid1['subconditions']) == 2 # TRANSFER divisible asset @@ -416,6 +420,7 @@ def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(b, user_pk, def test_muiltiple_in_mix_own_multiple_out_mix_own_transfer(b, user_pk, user_sk): from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details # CREATE divisible asset tx_create = Transaction.create([b.me], [([user_pk], 50), ([user_pk, b.me], 50)]) @@ -442,15 +447,15 @@ def test_muiltiple_in_mix_own_multiple_out_mix_own_transfer(b, user_pk, cond_cid0 = tx_transfer_signed.outputs[0].to_dict() cond_cid1 = tx_transfer_signed.outputs[1].to_dict() - assert 'subfulfillments' not in cond_cid0['condition']['details'] - assert 'subfulfillments' in cond_cid1['condition']['details'] - assert len(cond_cid1['condition']['details']['subfulfillments']) == 2 + assert 'subconditions' not in cond_cid0['condition']['details'] + assert 'subconditions' in cond_cid1['condition']['details'] + assert len(cond_cid1['condition']['details']['subconditions']) == 2 - ffill_fid0 = tx_transfer_signed.inputs[0].fulfillment.to_dict() - ffill_fid1 = tx_transfer_signed.inputs[1].fulfillment.to_dict() - assert 'subfulfillments' not in ffill_fid0 - assert 'subfulfillments' in ffill_fid1 - assert len(ffill_fid1['subfulfillments']) == 2 + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' not in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid1['subconditions']) == 2 # TRANSFER divisible asset diff --git a/tests/backend/mongodb-ssl/__init__.py b/tests/backend/mongodb-ssl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/backend/mongodb-ssl/certs/ca.crt b/tests/backend/mongodb-ssl/certs/ca.crt new file mode 100644 index 00000000..9c083efb --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/ca.crt @@ -0,0 +1,39 @@ +-----BEGIN CERTIFICATE----- +MIIGzjCCBLagAwIBAgIJAOgGsskqnC78MA0GCSqGSIb3DQEBCwUAMIGbMQswCQYD +VQQGEwJERTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xGDAWBgNV +BAoMD0JpZ2NoYWluREIgR21iSDEQMA4GA1UECwwHUk9PVC1DQTEbMBkGA1UEAwwS +VGVzdCBJbmZyYSBSb290IENBMSEwHwYJKoZIhvcNAQkBFhJkZXZAYmlnY2hhaW5k +Yi5jb20wHhcNMTcwNjEzMTQzNTU1WhcNMjcwNjExMTQzNTU1WjCBmzELMAkGA1UE +BhMCREUxDzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQK +DA9CaWdjaGFpbkRCIEdtYkgxEDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRl +c3QgSW5mcmEgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIu +Y29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuIgbqLOkJoFtnIKF +Pq4nMY/uyLt3YRiWyWJq68EuZ5rkoQDJOzaVYGgoJDUVxunT2/MVyAbc1MQN5WGa +NU5aQZnyYAgC7Ci/u/5YfgfHz4H+uLXm6rRz6bVRAt4WY5ZIHEtp+ThX+rDCs7pE +jcZxZdFjCbyrNdZtyDvhoHUwgKBiJ3b2373tq7rihNPeThABjkYOy2qStUnpNdiN +R9IrvSOAS/MzJVO3aoKkFLnKk0hD2Gjdh4hS2o3ZeF1TVHoBk6rA4I9szikYbCKa +SXAtF8CpUqnbThESM6+PfCfNRG+d+MEOi0jiMZzOrQLyG4bPRiec8ArTR5Bv1hsh +aKfRJONuSnX+40YBfhwTMpBqdbntereBlT8ahOJNZTbot5XVxSt69KZ2PZ99UpUl +WK+M8QLmjjwZEFCo4scGEXy1+6QcgTAgY2cN1NaDrO+7FNANGSi/dDXXJfWRauSd +kdUnn6uYioL6bwqZ2gvUYEKT7ezF2/PImmyCyZ01/ovLuhB2aT/1kd51/KLeuvZ2 +8yIu5YnKSKp7Bur8d7KrQc7mf/GHUw9Kvdjb4K7OOXimHZhCjQpxOtFYHOo+lEur +zHrsMwciBcJKGQzVnuhpDh7J+JDHKSSfJJlTuOuxvVGgzVgzCzbUn57F6C9Vs7g8 +Wk+ldudK+kn9kV18ncpWnwdZl6cCAwEAAaOCAREwggENMB0GA1UdDgQWBBRpx4WA +ZOaQQOkwaIgj0k277N+YmDCB0AYDVR0jBIHIMIHFgBRpx4WAZOaQQOkwaIgj0k27 +7N+YmKGBoaSBnjCBmzELMAkGA1UEBhMCREUxDzANBgNVBAgMBkJlcmxpbjEPMA0G +A1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdjaGFpbkRCIEdtYkgxEDAOBgNVBAsM +B1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5mcmEgUm9vdCBDQTEhMB8GCSqGSIb3 +DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tggkA6AayySqcLvwwDAYDVR0TBAUwAwEB +/zALBgNVHQ8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIBAEsHfo6/yp38egCYiYej +b8Dm674M7/neaL/QBLTxCgnpImxE4gHRZi1yWR4im5UL6gjZ7mM7AGL7Q1D/grG2 +3rc/WoSjBQPIe6gpuQwAni/iCaNI/QTgXFCx5fWZj/eIRF6ipKzxWkem3PdSdYUz +BoDuEZaKE7j4KOZDGD4UgN8igsNzmXHYH9nMFR4OERdIlTaHXmJvQ+5/yaazzzF0 +XXvbHSv5gHrnGoveDFNj11UJKwumG+L5UvKWsMe8zoS9YjvMJ9M9yUZ1WHjEdSuB +erEbbKnkv7FITyM4urGOS6Y6CjTjV8xG5IPxkHUQjEavjVcvbMaRr8vKf5rf2iuR +hDDYNknQf5zm5qmBq/cPC4dupKQyW58Kt5JkoY4Ok1zs4n9i6EFLUCHO3NaHsnfF +5iNg48DfI2ssk2HVGLyI8AiLl/IftvAP3OOAn6gW3twvwKK6m1Yfv822odEHv9oB +SDXlvbZhnwe8ZvNRa7QCiItzE/b/bh0+c1pk9M169qQAcum8OdwljS6XBzk2o0mv +cP6VD+UlutkEpOFW10m8QAcGHPVICSpBBSnry8fX90465BvurVLgYb5VJ7l7VTjn +7j99dO0MhE0OSfHONUcbf9+nyBYMkh2Gj+/N3zWd/F/COHeZveRoSdc37dEJOWjz +lEAdkN13aos01b6Xk9Dn4bf7 +-----END CERTIFICATE----- diff --git a/tests/backend/mongodb-ssl/certs/crl.pem b/tests/backend/mongodb-ssl/certs/crl.pem new file mode 100644 index 00000000..be129d6f --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/crl.pem @@ -0,0 +1,23 @@ +-----BEGIN X509 CRL----- +MIIDvzCCAacCAQEwDQYJKoZIhvcNAQELBQAwgZsxCzAJBgNVBAYTAkRFMQ8wDQYD +VQQIDAZCZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEYMBYGA1UECgwPQmlnY2hhaW5E +QiBHbWJIMRAwDgYDVQQLDAdST09ULUNBMRswGQYDVQQDDBJUZXN0IEluZnJhIFJv +b3QgQ0ExITAfBgkqhkiG9w0BCQEWEmRldkBiaWdjaGFpbmRiLmNvbRcNMTcwNjEz +MTQzNjU1WhcNMTcxMjEwMTQzNjU1WqCB1jCB0zCB0AYDVR0jBIHIMIHFgBRpx4WA +ZOaQQOkwaIgj0k277N+YmKGBoaSBnjCBmzELMAkGA1UEBhMCREUxDzANBgNVBAgM +BkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdjaGFpbkRCIEdt +YkgxEDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5mcmEgUm9vdCBD +QTEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tggkA6AayySqcLvww +DQYJKoZIhvcNAQELBQADggIBAGGHwjqvEayCkMzacIrhLlT97ra/5BGv9DIsVJUo +oEL+NuCl3lyd3lP+jr+cam+tqGJhsf43i7ZndmU4CKPS2WbZWENGSFcVIcNV05qT +YvGE62TpX74ZbFUAAsrZSyEGJFkREyrIwCc6b8O0Gr0BKCbnmlj/3XEhe9bsAu8m +bZiN6I1e89Wbz+nNzUi1cE2ZRRQgiTX4CFuvl9L37N4KvAHH1HJn6xzWx+VYP5xQ +vN1SK0SvsWHk3jiiyRazunWgJrdwmorqWApYHFybUNKw4B5btKe3ezl13ZXOIwDA +Ui6Fhi1jHj0yimZfieChD/bqGKEBFwrYp44ZRz52cg+YYcYzY72Rn6g7x8TsbNt0 +7h4jq2MEQ3We4zYEXFz7ZPxNLn7wYxx0x5h2E5vPaXXp5W/TzpCquSkpGSWF26OH +QAXaDOESAJV3e2oPFN8Wger3Oj7FTa0IZtne1aH/wnY0keDsVuvA8sHoy1Ylw72H +cv6D2ABEm4erAJ7n6BQ/unYXd+qKCYPLdxdOyd9lBGJhk6uN+Nzued/z8SjV3XEb +i6JvHwUEl/hwRWFF3k5vDaX4d5Z4kTIzO9+4ut44WRcgJ4zRd64ZKpGPEnTg3VsJ +oqQmwcfBvThKZmgrvUnvPzGYbA5LduQYRJ+elD5hGHBOEvayAof3FLKZZG8zNjip +lLJj +-----END X509 CRL----- diff --git a/tests/backend/mongodb-ssl/certs/test_bdb_ssl.crt b/tests/backend/mongodb-ssl/certs/test_bdb_ssl.crt new file mode 100644 index 00000000..dcb1949e --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/test_bdb_ssl.crt @@ -0,0 +1,133 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 2 (0x2) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=ROOT-CA, CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + Validity + Not Before: Jun 13 14:44:30 2017 GMT + Not After : Jun 11 14:44:30 2027 GMT + Subject: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=BigchainDB-Instance, CN=test-bdb-ssl/emailAddress=dev@bigchaindb.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:cb:8b:a5:98:f8:cb:ab:f0:c3:e3:8a:b1:92:ba: + c3:45:e0:1e:ed:d8:f2:a8:39:02:fd:8f:0f:e1:c9: + 9d:79:0c:38:38:df:a7:ef:6d:10:32:5a:1f:c8:d8: + ef:ea:a3:51:40:c3:a0:9b:67:f5:91:43:4f:05:fb: + b3:05:9a:01:47:88:53:2d:a0:67:fd:0e:1d:a3:9c: + de:1f:48:9e:e7:9f:6a:cc:04:d2:9e:36:90:e0:52: + 01:11:31:6f:db:5a:aa:4e:3d:83:5c:b8:31:7d:8d: + 06:8f:c1:f8:71:9e:71:a1:ee:54:8e:6c:77:5f:b4: + 69:4a:2d:df:8d:6a:d3:02:26:91:3a:2e:9a:58:61: + 6b:18:1a:ac:7b:c6:e4:b7:4d:ca:af:97:14:af:fa: + 16:87:78:50:98:d4:d1:50:3e:e6:d1:c2:d8:85:ee: + 06:5b:2d:43:8d:d8:3d:22:6d:28:59:52:44:79:e5: + 49:58:82:1a:0e:7f:06:80:85:79:52:1a:c1:c0:32: + d2:28:c7:b9:c0:67:9d:5c:b3:13:08:07:95:d6:91: + 87:6e:f8:53:7a:fa:67:d5:c8:07:91:d9:46:03:45: + 9e:b1:be:f7:78:fb:9a:a5:73:41:cf:b5:02:73:ed: + d5:a8:da:77:bd:3c:cf:e8:e5:dc:1f:cf:d6:93:e9: + 50:d4:76:f2:53:ec:a8:7d:7b:a7:84:4c:95:00:3e: + ab:f4:8e:0b:b1:2a:ef:7d:a1:66:d6:a1:f0:21:5c: + 0f:94:0a:12:de:82:65:55:14:47:37:61:cf:68:12: + 13:c1:f1:7b:14:5f:5c:ff:cf:b1:68:37:d6:75:5a: + 7d:cc:6c:22:e6:34:07:d1:2e:66:a7:6a:1e:9f:ee: + e9:b8:5d:da:a2:25:1b:00:70:a9:65:8d:66:54:42: + 49:85:fa:07:56:b4:77:26:af:70:4b:4f:ed:74:68: + 72:d4:f5:f9:ea:cc:23:a3:d6:8c:39:a2:79:f6:8c: + 64:4c:e3:75:17:86:6f:f1:e1:de:33:ec:28:89:e3: + 3b:a1:73:c9:da:57:fa:9c:cc:8b:51:63:10:26:f3: + 27:9a:c0:e2:67:2d:52:e7:41:a0:7e:6b:6d:7c:3d: + cc:4a:51:8a:62:c5:17:9c:88:c2:5f:38:a2:8d:ba: + bb:6f:82:11:e3:6c:ec:af:58:f8:06:b0:2d:02:4f: + dd:73:81:69:3f:cc:76:72:a1:db:73:43:8c:97:39: + 30:49:d2:9a:77:30:49:21:85:32:0a:6a:37:bf:09: + 06:60:a3:0f:e5:ba:f5:07:2a:34:e5:3b:07:1d:10: + c1:c0:38:bc:95:dc:81:b2:89:ab:d5:17:9c:21:c3: + 1a:b2:61 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + X509v3 Subject Key Identifier: + B7:F5:E7:0E:F8:D8:FE:A1:56:5B:EA:80:2F:18:71:C2:44:0C:91:D0 + X509v3 Authority Key Identifier: + keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98 + DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + serial:E8:06:B2:C9:2A:9C:2E:FC + + X509v3 Extended Key Usage: + TLS Web Client Authentication + X509v3 Key Usage: + Digital Signature + Signature Algorithm: sha256WithRSAEncryption + 78:44:00:be:10:3b:f3:40:e1:5e:e4:3a:64:99:13:71:1d:91: + 96:f6:f1:0f:db:99:6c:65:c3:be:c9:0d:d7:a1:c8:7c:09:e6: + 56:5b:32:44:5f:e8:00:27:b5:20:28:d9:19:5a:74:21:4f:1a: + ef:5a:e9:cc:f4:97:f1:9f:97:9b:45:35:cb:df:27:6a:75:ce: + 9e:0e:11:be:03:fa:1a:91:77:9d:7d:6a:76:59:6b:98:96:09: + 21:cd:ca:54:1e:1f:75:58:68:5d:af:c2:8a:18:c5:56:d9:56: + 39:c6:a7:2a:a4:0e:0b:88:7e:55:72:7f:ec:07:0d:7f:7a:c0: + 14:8f:44:f4:cc:3b:30:97:8a:98:e2:da:7e:88:b8:a5:93:4c: + f4:92:e1:e8:84:60:bc:f9:e4:55:0f:68:ba:34:70:4f:9f:47: + 63:c1:2f:96:78:ab:43:80:87:f2:0d:10:57:a0:a0:8c:d4:93: + c3:89:ef:f0:2f:58:63:53:8c:1e:29:4c:a5:88:ec:56:af:22: + 65:54:77:6c:f8:cd:68:2d:34:f7:71:cf:12:6e:ba:50:8f:30: + a0:05:31:e7:32:27:29:e5:1b:a9:40:3c:49:45:a3:8e:2d:10: + 0b:b4:da:f5:73:e7:aa:d1:c7:a8:a6:f5:32:4a:33:f3:60:3d: + 72:4e:b9:1e:15:e9:7d:0c:a9:f8:57:72:2b:60:24:18:47:5b: + 34:f5:25:ef:93:10:4a:0b:ed:e8:39:2e:d8:9e:bd:32:67:ce: + 7c:c7:a4:0e:5f:03:1e:8d:4a:7f:ac:7f:4e:7b:f8:26:44:1c: + 9f:6b:a0:9e:4d:90:31:13:8a:46:5f:87:9e:bc:06:f2:b6:e5: + 6b:75:d1:f3:c0:4d:fe:c5:16:34:35:ce:6e:31:f3:1f:cd:4e: + 13:5d:0a:84:00:cc:72:b5:ef:a4:90:74:70:53:9b:6c:b3:58: + 5e:3b:ba:5b:ff:4c:fe:47:7e:20:1c:83:04:57:7a:a5:08:ed: + 29:51:11:e0:a5:81:92:b5:4f:32:74:35:be:8a:c6:82:7a:50: + 45:f9:ee:57:62:a5:41:57:dc:3d:f7:bd:17:59:2d:53:2c:d0: + 81:76:e8:1b:64:bd:80:94:eb:b1:f6:0f:8f:c7:50:cb:c4:c2: + 33:b0:78:78:d4:61:d2:d7:54:0a:71:24:59:0f:30:23:8c:45: + d6:b9:f1:5c:99:eb:20:11:2f:ca:36:39:36:72:e9:f5:24:47: + 54:54:20:4d:1d:aa:cd:ec:ec:4b:89:2b:67:00:62:64:2c:05: + 19:6c:91:72:01:bb:04:0c:f0:e1:27:5e:c9:9b:f1:41:09:8a: + dc:62:85:a0:87:c8:d5:ab +-----BEGIN CERTIFICATE----- +MIIG3jCCBMagAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBmzELMAkGA1UEBhMCREUx +DzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdj +aGFpbkRCIEdtYkgxEDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5m +cmEgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tMB4X +DTE3MDYxMzE0NDQzMFoXDTI3MDYxMTE0NDQzMFowgaExCzAJBgNVBAYTAkRFMQ8w +DQYDVQQIDAZCZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEYMBYGA1UECgwPQmlnY2hh +aW5EQiBHbWJIMRwwGgYDVQQLDBNCaWdjaGFpbkRCLUluc3RhbmNlMRUwEwYDVQQD +DAx0ZXN0LWJkYi1zc2wxITAfBgkqhkiG9w0BCQEWEmRldkBiaWdjaGFpbmRiLmNv +bTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMuLpZj4y6vww+OKsZK6 +w0XgHu3Y8qg5Av2PD+HJnXkMODjfp+9tEDJaH8jY7+qjUUDDoJtn9ZFDTwX7swWa +AUeIUy2gZ/0OHaOc3h9InuefaswE0p42kOBSARExb9taqk49g1y4MX2NBo/B+HGe +caHuVI5sd1+0aUot341q0wImkToumlhhaxgarHvG5LdNyq+XFK/6Fod4UJjU0VA+ +5tHC2IXuBlstQ43YPSJtKFlSRHnlSViCGg5/BoCFeVIawcAy0ijHucBnnVyzEwgH +ldaRh274U3r6Z9XIB5HZRgNFnrG+93j7mqVzQc+1AnPt1ajad708z+jl3B/P1pPp +UNR28lPsqH17p4RMlQA+q/SOC7Eq732hZtah8CFcD5QKEt6CZVUURzdhz2gSE8Hx +exRfXP/PsWg31nVafcxsIuY0B9EuZqdqHp/u6bhd2qIlGwBwqWWNZlRCSYX6B1a0 +dyavcEtP7XRoctT1+erMI6PWjDmiefaMZEzjdReGb/Hh3jPsKInjO6FzydpX+pzM +i1FjECbzJ5rA4mctUudBoH5rbXw9zEpRimLFF5yIwl84oo26u2+CEeNs7K9Y+Aaw +LQJP3XOBaT/MdnKh23NDjJc5MEnSmncwSSGFMgpqN78JBmCjD+W69QcqNOU7Bx0Q +wcA4vJXcgbKJq9UXnCHDGrJhAgMBAAGjggEjMIIBHzAJBgNVHRMEAjAAMB0GA1Ud +DgQWBBS39ecO+Nj+oVZb6oAvGHHCRAyR0DCB0AYDVR0jBIHIMIHFgBRpx4WAZOaQ +QOkwaIgj0k277N+YmKGBoaSBnjCBmzELMAkGA1UEBhMCREUxDzANBgNVBAgMBkJl +cmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdjaGFpbkRCIEdtYkgx +EDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5mcmEgUm9vdCBDQTEh +MB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tggkA6AayySqcLvwwEwYD +VR0lBAwwCgYIKwYBBQUHAwIwCwYDVR0PBAQDAgeAMA0GCSqGSIb3DQEBCwUAA4IC +AQB4RAC+EDvzQOFe5DpkmRNxHZGW9vEP25lsZcO+yQ3Xoch8CeZWWzJEX+gAJ7Ug +KNkZWnQhTxrvWunM9Jfxn5ebRTXL3ydqdc6eDhG+A/oakXedfWp2WWuYlgkhzcpU +Hh91WGhdr8KKGMVW2VY5xqcqpA4LiH5Vcn/sBw1/esAUj0T0zDswl4qY4tp+iLil +k0z0kuHohGC8+eRVD2i6NHBPn0djwS+WeKtDgIfyDRBXoKCM1JPDie/wL1hjU4we +KUyliOxWryJlVHds+M1oLTT3cc8SbrpQjzCgBTHnMicp5RupQDxJRaOOLRALtNr1 +c+eq0ceopvUySjPzYD1yTrkeFel9DKn4V3IrYCQYR1s09SXvkxBKC+3oOS7Ynr0y +Z858x6QOXwMejUp/rH9Oe/gmRByfa6CeTZAxE4pGX4eevAbytuVrddHzwE3+xRY0 +Nc5uMfMfzU4TXQqEAMxyte+kkHRwU5tss1heO7pb/0z+R34gHIMEV3qlCO0pURHg +pYGStU8ydDW+isaCelBF+e5XYqVBV9w9970XWS1TLNCBdugbZL2AlOux9g+Px1DL +xMIzsHh41GHS11QKcSRZDzAjjEXWufFcmesgES/KNjk2cun1JEdUVCBNHarN7OxL +iStnAGJkLAUZbJFyAbsEDPDhJ17Jm/FBCYrcYoWgh8jVqw== +-----END CERTIFICATE----- diff --git a/tests/backend/mongodb-ssl/certs/test_bdb_ssl.key b/tests/backend/mongodb-ssl/certs/test_bdb_ssl.key new file mode 100644 index 00000000..36e91e9e --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/test_bdb_ssl.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDLi6WY+Mur8MPj +irGSusNF4B7t2PKoOQL9jw/hyZ15DDg436fvbRAyWh/I2O/qo1FAw6CbZ/WRQ08F ++7MFmgFHiFMtoGf9Dh2jnN4fSJ7nn2rMBNKeNpDgUgERMW/bWqpOPYNcuDF9jQaP +wfhxnnGh7lSObHdftGlKLd+NatMCJpE6LppYYWsYGqx7xuS3TcqvlxSv+haHeFCY +1NFQPubRwtiF7gZbLUON2D0ibShZUkR55UlYghoOfwaAhXlSGsHAMtIox7nAZ51c +sxMIB5XWkYdu+FN6+mfVyAeR2UYDRZ6xvvd4+5qlc0HPtQJz7dWo2ne9PM/o5dwf +z9aT6VDUdvJT7Kh9e6eETJUAPqv0jguxKu99oWbWofAhXA+UChLegmVVFEc3Yc9o +EhPB8XsUX1z/z7FoN9Z1Wn3MbCLmNAfRLmanah6f7um4XdqiJRsAcKlljWZUQkmF ++gdWtHcmr3BLT+10aHLU9fnqzCOj1ow5onn2jGRM43UXhm/x4d4z7CiJ4zuhc8na +V/qczItRYxAm8yeawOJnLVLnQaB+a218PcxKUYpixReciMJfOKKNurtvghHjbOyv +WPgGsC0CT91zgWk/zHZyodtzQ4yXOTBJ0pp3MEkhhTIKaje/CQZgow/luvUHKjTl +OwcdEMHAOLyV3IGyiavVF5whwxqyYQIDAQABAoICAQC0NvMqanWhyW0WxJCOMi98 +aX/Y5LGMAdZE+2p7ZY+4QfpxWIyOrsidXDAyGujonSrupYZIshW5RJft2zlY507k +r629gv0vD1VsrBH6LskKmJntAwQbsoI5KkHpoBTCaRbKaSoqaKbg24EIvRukNT20 +uphA9YYHxWtHkzAQPJmQmOcSVXqwb6hrUgqPGL0eEpE3QBO/7zQNQ5hQiHS+kMK6 +y8H7apBUH9Jk/yS5m14TH4SjTD/RTneViCAQoJyW9ju/WP+7vYIX+WVZUu+xt02S +CY2Y0eLPym8u14jkODQF55knFMZ1Zoc4n8CQD1qZm+UFOFC1yubl9MYdoF4C3ZuC +vABnU7UU6pSABJ3glcwWUrGPxgbWTSlCTFwJqWeUZfEwWXQPhfoGIfckOf89tDsB +9TUd8fnfKVUrqDnEVPryvMZipG1nX4pZX/nXM/22sepdyuQ21quCcUTcUGwBhhQA +gfVJMPUc2dilJyYILYgmZ9k3J1iFVwIahMtqxY7oSaQi5bD9Pcst9MzFwiGo7T2q +4CjfUTPCiQeE+kyiEo1HR6Mhzd80Nobh0BQfMuauBMmkMlA3HPeztyR2U16QdDIa +LG2DOl+Ak4OultjKtGKovsb9FnzDnr/N+ONLgRyKPMwjLgjRPBpGmxSltjfqHpC/ +72OK3QPbDe7FRa3+jzC8ZQKCAQEA+RA7fhUEQv722SVpJI1pEvyQGCede65VB3lp +tN4wWqXfdKBKzTcxx6xkojvZPQe5NQFK3743Ui5qbgNxnxuqU/+KsBngZBVAYV6k +muCmpJsucCXVULeG5Cei7Z2FoRn2zjI1+7cMX9kccq+lGjy3imJPjhy9ChTWt/GB +P4Ii6ow2f+ZNKd5Vs/BRvGurLdCWg47UTzujHWtI24nJHjYfWpGIdAmJ9cuA/QgJ +RCVs8U0vXzcRKwGkn45q9t7A9LMMZ2OIryhN3PPY4p1i4zu4kwFEQeJ1CCSrAiTn +etitPxiovo+dmCFWxXeYuY+PNsi5C6kwOHVPDYlv+YrP0cm3jwKCAQEA0TbhuDaQ +rkuVIIlzrQzLl0KqtoQ/p+5KI4WmzDdpmWvLhAGug7Xfi5AmSAmJB+ui/XeHCjyg +Vy+nyoxzJrdjg0WShRgRx2b3rYqxTDsV72sbl1ofrCyX9DOW2XaiLCXPkGQfOmkS +yYagR3xyfsVsf44bwUWj2vckXFhZFZDR2DpRWL20S852RpXvwzOoOWhfaBAqHeR+ +qBWVsDuRwo0HCkidgBYnRpg4FAitoiVM+MpB3r84pOK4bO6s0n384l1iFLhWli/d +6hLVa4Hpec5jMHQajhOjdk03gK0xmS9Trcv1aUboih7KH0FlbdlWeKWi/F7sT7P9 +3MFrs9G+Ybd/DwKCAQEAiA1t8jnY0iAlCAl/nhABTfXZYNiDFoTsveB89ehJ0bq6 +jiLhuahk8QYjEtxOlyAY8/N4yzBFWAcy7FXFQ30BVlFJmVkJUqKpQIdKs6/0WAbN +H0YLeMRIU2mzfsmFrbNZNtoG0zHy+IjJGX0JW4O/X0DE5ISeX1tyz5iFWgPkvutI ++iT8Edr6RgkJHTxTjftbvRHQuFeDixaH/iUOUAqQphDJ0VlEm5hHJfG13hvznvzi +28hhAHUtUFuj4qdcEE+efvvINqZ1ojUyTNhcpHSDJwqSFst10rT1FX7DVD/4o3b8 +9tvM4sWTeNsT+omJWgvbyZrqVG0AegyRtmc1jwv50QKCAQAjNGz/Jo9qBOXvN7Hz +aLfJmDpJxC9B/uMS3yaWLqziWDVC3VWUbEJVpcgLTy9z2b7vj/F1U4ZXpXlCqCeo +WCuypz5kjwxO1ZDP7wqunTKvuwJFkbDKtCZNvXCg30mizCoFThPDLHMw3iqSXOqX +UnlTbYJWH3XXHsdLksJK0re8/vM5T5LLPs7ASfKykjq4jkufozizFwQJHLi3kw6Q +AlMw888tAo1RX8Pv+xXctxIgu0giR2Msu8n06qTCNtka7kPW4L4RP7TD8q9fC9lR +2dzvBlqBleRnv86bJIm4ETKviCaftILk2xF/+O0scuoOGzE17nMtZkhNoW6SL7Ut +lEcZAoIBAGMtuS1SJAnJ2wt5NY3zLAIoTwOr4LQLF1ukF+ogkck/GsRaJxGQRaE/ +tupQBRzsZN3MxjRspDzH+583yIEWFWwxDzDbz8P9PuU+Dgm3vRGJ+qvzTDHvnPzd +3mTOMnNMjchi6wwV9t/SxwhtnUoIHG0FtrXE+xH4QFnXaufGqw8rWbuPRjydH9iJ +8h+U4g4oaRpvftDHZ6HUMYgqC3/BtB9yk0CQ/BnTG1FlZZw6Ybu4SNtok2QrR1xN +MdcSNvQrgkMbVJ8ysKOwJNol3yw5fqYUz6KRxzU28MdOyELhcWhWUC9/D+5ib13f +Y78+RqMxuQBGGFp1ahS075xikXeb0TE= +-----END PRIVATE KEY----- diff --git a/tests/backend/mongodb-ssl/certs/test_mdb_bak_ssl_cert_and_key.pem b/tests/backend/mongodb-ssl/certs/test_mdb_bak_ssl_cert_and_key.pem new file mode 100644 index 00000000..da457e71 --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/test_mdb_bak_ssl_cert_and_key.pem @@ -0,0 +1,185 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4 (0x4) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=ROOT-CA, CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + Validity + Not Before: Jun 14 12:45:09 2017 GMT + Not After : Jun 12 12:45:09 2027 GMT + Subject: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=MongoDB-Bak-Instance, CN=test-mdb-bak-ssl/emailAddress=dev@bigchaindb.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:9e:49:0d:d6:44:06:db:ad:bd:24:0c:d4:d2:f6: + 0e:e9:14:5a:52:b7:3d:72:97:ae:1c:b4:dd:6c:a6: + 2b:46:94:a9:60:29:8a:15:75:3d:35:66:a1:7e:3c: + a7:09:38:4d:30:4a:5f:d1:01:22:a5:b0:f0:43:ed: + 9e:e8:6d:b5:4b:da:3d:50:d7:67:c3:bc:a2:08:72: + 2f:fe:18:54:2a:2e:a8:6b:f4:ca:fb:4a:50:f0:48: + c3:97:62:b7:f5:a0:a2:50:2b:c5:b3:ed:6f:b4:86: + 09:a6:67:68:1f:28:69:d4:0e:73:c1:a2:6c:25:a5: + 55:37:8c:7e:a4:9e:aa:83:8d:9c:b6:29:19:6f:e9: + 86:58:9f:34:8b:92:39:9a:4d:ac:2f:1b:ee:43:a8: + 25:dc:d6:82:63:65:e6:f9:71:ea:69:ac:4f:1b:9f: + 96:ba:21:88:db:7b:87:4a:5c:84:4a:d6:39:3c:1f: + ea:e7:3a:9e:e0:31:32:e9:3d:48:da:0d:6d:47:74: + 2c:58:e6:ad:65:10:b8:64:7b:80:cb:b0:a2:f4:a6: + 16:27:b0:84:6e:09:c6:30:a3:b7:fa:34:7a:96:5d: + 61:71:7d:7d:dc:c8:69:9d:4c:2f:b6:a1:20:31:99: + b1:96:9a:9e:be:f4:ec:da:2a:6c:3a:0a:e4:94:ef: + 67:a5:f4:7c:ae:15:f2:67:8b:f4:f4:18:32:1e:7f: + 87:79:e7:87:a0:74:99:57:f2:44:62:fe:93:93:21: + 13:b5:98:dd:fb:98:67:e0:8f:e3:19:36:0b:9e:5b: + 67:a0:37:77:62:78:9b:6c:be:79:13:bd:79:ae:34: + b7:92:f1:8f:17:9c:0b:6a:42:9a:ed:23:e4:71:0d: + e6:f3:6d:9c:58:54:88:2f:ed:85:a3:5c:a4:38:6d: + a3:b9:bc:ba:56:ad:f8:2c:fa:8c:e8:83:de:1b:af: + 11:88:e9:81:08:c8:d4:03:68:d4:e7:11:c7:e6:1d: + 93:7f:02:2c:3d:42:e3:bb:f1:68:70:21:95:87:db: + c5:c8:43:64:d5:d9:10:94:cb:e4:17:e3:5c:21:38: + fb:9c:96:69:da:24:e3:59:e8:d9:f3:41:45:04:8a: + 04:c8:bd:04:85:7a:9c:72:9f:5d:34:38:1d:1c:26: + 85:6c:c3:1c:6d:df:6b:44:a4:ac:f7:27:0d:8f:1a: + 9b:d2:53:5c:15:bd:1e:f1:de:f1:45:d7:96:d0:50: + 0a:43:3b:53:ea:1a:8d:67:ad:68:d0:57:dc:3c:f5: + 63:fb:0e:ff:b2:cf:59:30:d5:12:bc:2c:62:00:cf: + c3:ae:ea:20:04:c1:67:e0:3f:92:99:e5:04:cc:7b: + a6:7f:b9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + X509v3 Subject Key Identifier: + 95:F3:A7:FB:99:C6:9D:91:36:06:51:31:39:EC:37:42:89:07:AB:31 + X509v3 Authority Key Identifier: + keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98 + DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + serial:E8:06:B2:C9:2A:9C:2E:FC + + X509v3 Extended Key Usage: + TLS Web Client Authentication + X509v3 Key Usage: + Digital Signature + Signature Algorithm: sha256WithRSAEncryption + 5b:42:f5:e9:cc:2a:40:8a:53:29:d9:67:2b:5d:df:25:b8:08: + 6f:6f:0f:d4:e1:b7:60:eb:d4:04:d4:3c:49:62:a5:78:59:48: + d6:4b:c3:24:04:86:51:99:01:6f:f5:ce:a1:b8:c0:d5:56:4a: + 23:86:f9:22:e6:42:3b:39:8b:66:64:21:f6:72:6b:77:79:4b: + 9f:3e:ec:0e:ba:cf:bd:72:73:02:66:bf:cf:e8:b2:75:ee:07: + 28:ae:26:98:b8:40:ec:dc:d5:12:27:27:34:3e:4f:55:b6:36: + e0:3b:58:ec:2d:fa:59:e3:c1:ec:16:93:8d:72:f6:ad:f8:dd: + 59:6e:c2:cb:51:82:f1:fc:b6:7e:67:61:f7:81:76:9d:a7:83: + 52:06:cb:b7:fe:52:f4:2a:bc:62:66:16:4b:bd:03:13:7f:e0: + f1:7e:c4:67:e4:9a:d4:1f:bf:a2:a1:f9:2a:8b:bd:d1:06:35: + 16:97:7b:93:fa:3e:e0:df:4f:60:60:74:ef:18:0c:69:10:61: + 17:4a:8b:d7:4d:0b:83:6c:de:c3:ca:34:ad:02:35:34:e5:2c: + 15:28:4c:ff:5b:e7:27:eb:87:c9:88:21:3e:ed:b3:4e:cc:80: + 2f:fe:87:e4:c7:d8:7c:5d:61:79:db:49:bc:f6:60:28:97:0d: + 17:0e:f4:7d:3a:ca:bc:d1:f2:62:70:a9:19:8b:f8:74:1e:c4: + 10:f9:7a:62:d0:65:d3:00:f4:3d:08:11:5c:d5:d0:97:3e:52: + 0c:51:1a:e6:71:bf:d9:25:c7:38:b7:d1:17:04:c3:a3:74:34: + 51:7e:3d:78:3f:e7:c9:e7:e2:37:db:33:43:26:ca:7f:2c:d5: + cd:fc:55:2d:3f:1e:7b:95:af:44:ca:b6:9f:0e:02:d3:62:e6: + 1f:96:a6:b2:0d:de:0b:31:b6:4f:de:a8:63:85:8c:c2:5d:89: + f9:ba:b2:e9:41:19:60:3b:06:18:c5:f6:9f:8c:f9:fa:36:18: + 16:3b:c4:8a:60:5e:7c:06:8c:f3:3a:c0:25:bc:3f:fc:f1:5d: + a3:81:a4:6a:48:05:f3:0b:cb:f7:45:87:4b:32:5f:b2:d4:5c: + 85:36:ec:3f:aa:23:81:fe:ce:75:7d:54:12:87:b0:95:a7:57: + 81:c5:4b:f3:d9:9a:d2:fb:af:bb:a7:6a:b2:23:92:1d:28:8f: + a7:21:bd:3c:21:fb:39:fd:73:06:84:d2:9c:6b:06:c9:3f:22: + 9d:dc:a8:74:9b:76:8d:e3:09:9d:ef:02:18:9b:1e:52:69:eb: + be:1b:bb:73:e2:36:06:4b:27:ad:0f:87:66:cc:36:81:5a:55: + e1:7b:7b:d0:4d:2f:55:95 +-----BEGIN CERTIFICATE----- +MIIG4zCCBMugAwIBAgIBBDANBgkqhkiG9w0BAQsFADCBmzELMAkGA1UEBhMCREUx +DzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdj +aGFpbkRCIEdtYkgxEDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5m +cmEgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tMB4X +DTE3MDYxNDEyNDUwOVoXDTI3MDYxMjEyNDUwOVowgaYxCzAJBgNVBAYTAkRFMQ8w +DQYDVQQIDAZCZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEYMBYGA1UECgwPQmlnY2hh +aW5EQiBHbWJIMR0wGwYDVQQLDBRNb25nb0RCLUJhay1JbnN0YW5jZTEZMBcGA1UE +AwwQdGVzdC1tZGItYmFrLXNzbDEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWlu +ZGIuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnkkN1kQG2629 +JAzU0vYO6RRaUrc9cpeuHLTdbKYrRpSpYCmKFXU9NWahfjynCThNMEpf0QEipbDw +Q+2e6G21S9o9UNdnw7yiCHIv/hhUKi6oa/TK+0pQ8EjDl2K39aCiUCvFs+1vtIYJ +pmdoHyhp1A5zwaJsJaVVN4x+pJ6qg42ctikZb+mGWJ80i5I5mk2sLxvuQ6gl3NaC +Y2Xm+XHqaaxPG5+WuiGI23uHSlyEStY5PB/q5zqe4DEy6T1I2g1tR3QsWOatZRC4 +ZHuAy7Ci9KYWJ7CEbgnGMKO3+jR6ll1hcX193MhpnUwvtqEgMZmxlpqevvTs2ips +OgrklO9npfR8rhXyZ4v09BgyHn+HeeeHoHSZV/JEYv6TkyETtZjd+5hn4I/jGTYL +nltnoDd3YnibbL55E715rjS3kvGPF5wLakKa7SPkcQ3m822cWFSIL+2Fo1ykOG2j +uby6Vq34LPqM6IPeG68RiOmBCMjUA2jU5xHH5h2TfwIsPULju/FocCGVh9vFyENk +1dkQlMvkF+NcITj7nJZp2iTjWejZ80FFBIoEyL0EhXqccp9dNDgdHCaFbMMcbd9r +RKSs9ycNjxqb0lNcFb0e8d7xRdeW0FAKQztT6hqNZ61o0FfcPPVj+w7/ss9ZMNUS +vCxiAM/DruogBMFn4D+SmeUEzHumf7kCAwEAAaOCASMwggEfMAkGA1UdEwQCMAAw +HQYDVR0OBBYEFJXzp/uZxp2RNgZRMTnsN0KJB6sxMIHQBgNVHSMEgcgwgcWAFGnH +hYBk5pBA6TBoiCPSTbvs35iYoYGhpIGeMIGbMQswCQYDVQQGEwJERTEPMA0GA1UE +CAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xGDAWBgNVBAoMD0JpZ2NoYWluREIg +R21iSDEQMA4GA1UECwwHUk9PVC1DQTEbMBkGA1UEAwwSVGVzdCBJbmZyYSBSb290 +IENBMSEwHwYJKoZIhvcNAQkBFhJkZXZAYmlnY2hhaW5kYi5jb22CCQDoBrLJKpwu +/DATBgNVHSUEDDAKBggrBgEFBQcDAjALBgNVHQ8EBAMCB4AwDQYJKoZIhvcNAQEL +BQADggIBAFtC9enMKkCKUynZZytd3yW4CG9vD9Tht2Dr1ATUPElipXhZSNZLwyQE +hlGZAW/1zqG4wNVWSiOG+SLmQjs5i2ZkIfZya3d5S58+7A66z71ycwJmv8/osnXu +ByiuJpi4QOzc1RInJzQ+T1W2NuA7WOwt+lnjwewWk41y9q343VluwstRgvH8tn5n +YfeBdp2ng1IGy7f+UvQqvGJmFku9AxN/4PF+xGfkmtQfv6Kh+SqLvdEGNRaXe5P6 +PuDfT2BgdO8YDGkQYRdKi9dNC4Ns3sPKNK0CNTTlLBUoTP9b5yfrh8mIIT7ts07M +gC/+h+TH2HxdYXnbSbz2YCiXDRcO9H06yrzR8mJwqRmL+HQexBD5emLQZdMA9D0I +EVzV0Jc+UgxRGuZxv9klxzi30RcEw6N0NFF+PXg/58nn4jfbM0Mmyn8s1c38VS0/ +HnuVr0TKtp8OAtNi5h+WprIN3gsxtk/eqGOFjMJdifm6sulBGWA7BhjF9p+M+fo2 +GBY7xIpgXnwGjPM6wCW8P/zxXaOBpGpIBfMLy/dFh0syX7LUXIU27D+qI4H+znV9 +VBKHsJWnV4HFS/PZmtL7r7unarIjkh0oj6chvTwh+zn9cwaE0pxrBsk/Ip3cqHSb +do3jCZ3vAhibHlJp674bu3PiNgZLJ60Ph2bMNoFaVeF7e9BNL1WV +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCeSQ3WRAbbrb0k +DNTS9g7pFFpStz1yl64ctN1spitGlKlgKYoVdT01ZqF+PKcJOE0wSl/RASKlsPBD +7Z7obbVL2j1Q12fDvKIIci/+GFQqLqhr9Mr7SlDwSMOXYrf1oKJQK8Wz7W+0hgmm +Z2gfKGnUDnPBomwlpVU3jH6knqqDjZy2KRlv6YZYnzSLkjmaTawvG+5DqCXc1oJj +Zeb5cepprE8bn5a6IYjbe4dKXIRK1jk8H+rnOp7gMTLpPUjaDW1HdCxY5q1lELhk +e4DLsKL0phYnsIRuCcYwo7f6NHqWXWFxfX3cyGmdTC+2oSAxmbGWmp6+9OzaKmw6 +CuSU72el9HyuFfJni/T0GDIef4d554egdJlX8kRi/pOTIRO1mN37mGfgj+MZNgue +W2egN3dieJtsvnkTvXmuNLeS8Y8XnAtqQprtI+RxDebzbZxYVIgv7YWjXKQ4baO5 +vLpWrfgs+ozog94brxGI6YEIyNQDaNTnEcfmHZN/Aiw9QuO78WhwIZWH28XIQ2TV +2RCUy+QX41whOPuclmnaJONZ6NnzQUUEigTIvQSFepxyn100OB0cJoVswxxt32tE +pKz3Jw2PGpvSU1wVvR7x3vFF15bQUApDO1PqGo1nrWjQV9w89WP7Dv+yz1kw1RK8 +LGIAz8Ou6iAEwWfgP5KZ5QTMe6Z/uQIDAQABAoICADsiR80YtQc1LUhektQNoRxq +wiPM2WQKTr3ixCZnHhvMRkrque+yUR+2K/chabYEFrJH1uwaZHtKUzjNeWSUN/sS +mX2uO9HgkxhcsJlZNXhc3gcW+Q5QgVSDmq7f7qIRVRJmiAHkXqsuuEQ2tauOSZsz +mwNoTsbxsZiDIH0X2dQz/6v+RsaMk+hf8h2Cj4qaIg2nfahBFYQfj67azyO57z1z +ZkqHoKQBC0QULkMVtUbQKanQss2YFOrk9oQ0pRbxlTXwrPC5nWX4jSrdCQ0P/JEx +y20ggRkrBaP+Rilvmay7TkA8Bd2J8gsV/21XXNOq+7GsKkpk7mfDGZPFpggOOMYs +xmp9rCZ+/0wpGQ130/ks3bQRHeVXQ0WuEXrmkdHhyyRycYocDfUhC4YB5NJOhI8H +Xl/ScebL3xwl56CNbfdyG6VeAqpJk6qxoklMw+zWJQeHtaHkXObWSSORf0KCAHV7 +5/FWH7QP93g3X4r9Cq0zVI0e9ImC81azxj335bWpZi63YTvZriStak2ucfkBPXof +zTQHVi47E8fOG6HjEQYLM/FohXRxSKodBruKEKZlzqGLaSVlfj25v1l61mS4Owjj +2VWEuCraGixRIfawK6CtuS6ZRCLZyuVISul0Xhmm9EJ9rrqaJS3rCHp40S/prnFY +TACkRoLkBqftkva2uVxBAoIBAQDQK0hEZ3jVznmybNyvscdkYnJHFqJqjQgR3DKM +1RxcOpsLcEJZv5Xw9yze1QndCplnWsHsbOYI/Bz6Bnf60DRnbiZrftNEkabEgPkE +pEnZTCWm/ynTjhJy5y5ttnrqx2CHUpLoVnwqQKQqp3p/gjXQcp9NqX8/ieA4ae63 +tPMpQERlA3ETHprZ26NkjIl35oO2Hl/vw80Inh26EHFlKZkzKRsCaqBLgIkDpbOe +vgziyBUCk3pmyKI62pPu4S3Xy7TaYtjUeZvwUI8u3hvE6v2Qjj9nS6qvTOGRg/hG +3DSBEqXUxMTYu+41zPyYZiSoUpFkReV9G/K3wO9b0Y+po+NFAoIBAQDCp4+uwo47 +90S79J2wM4nEhtPvgg5Zvv+8a1Uedng+DKokmPdcqD9RSREcbuj2BOwS/phTePKh +0UFgy43B8DrTDNrYvZPTjPalxo8ks122+vzM+Y43le072wOQkHfVZP/Y1+lOOsVf +TjGrp82keg4w784w7OuJNg+i7+rwsOwBn6iavHbHi3c8IGXDsNe5+LEwTSXSyxiF +s/NFBYOxNDV1GVATTbyi5bEKLDtrD1GXRtCU4atGf9fwYSU08KMjrhRQLP+0sYwN +lGSpJnJc7M7CuikxT646+ENTZC1vZlp1bUwsX7sjk5kxq6fkf0LM0/7hLg0yPOFV +XTBeLPYJrRflAoIBABYbgqFBG+QY+XOpfAZsqvBORDSogrcuIx1CdVvfhhOFZdHh +Kiq93f6pQWqo1VNUPNZtHQsuxX9OxwUGitdoJSiW2h2wal9t2Hgl+kwz5mPdYmJJ +1vdQr0TkqFmed49XfTjh3Bgwlx8lnkmpX6kK+wwYIDLvPURSMC3NkjyQSwKmgJz5 +sJiHN4rLeJ7FDhRdtr8wmp+r+6peoGmSNXi44jw8sVGgYUWjcOmiP2EsbdHnzdNo +NieURyZY8Dz+TRAN4jcPKFfoUldDNvGTRP+0tRkVAkbGEmprj8kUatmbqTfL8zCV +dRJPYwzzqB/HOT1nB2nOwoB251/8bW8i4k7xyMECggEANOd9sa3HIp0t3Te6s+0O +AI/0giAC7nlu4DL7y2+/dn3SsGeys0g3DUyijevG/TaRQwhXNjilGT3aWwh32sID ++uB/inDcAJ9LWfsBZKQrUFQe6UbaVFk3RRFdgqkBKMpujuz/x/dJNYH3FgRha9aQ +jNRgYgPCcGR1E3/JhulO+5H9LTETx2AsY/caMXma6DyjS9Fr+kKgw5YJBDVfeYYL +EWxzywtRvaRX3b/v1kUvk8H3Zr9+4YZVlkuQ5TaR4FyrwK66QM4QlpBCW4bLhl4G +Q/58u55AaF6ZTczoXGKhK1EtZtIN9rli5ZEV7JB6A1mK6ICvrXvGcoEFaFMn+7FD +0QKCAQB+zcumaZJ2f70TUc0cSpH5ObEdQzgLDgYk11ntogFWaXefY84R2UOAnVmT +TOAdl5CHY+iBwYVldGO9/+lSaeXzsFYeuvkoVPg8jnSoocc8H+QdgCjXwIPtalAJ +DmMo5OehmmbeZl/HYnESodqy+94DylxpoAwRwPh4m4H28lJObOkc+aP94Ij+i403 +PW9HxSK4u1OTSAXUHrsHxYTHOoO0KIx7tDQtgToWJiXIF4OcvHpVrYhZlmX8jdZB +rzfGm7L2NbXOxMyRwzkiP2u3Jy2KMDjkzzrcj/baBASw/gvTBNg0Av+hxDLHAVKc ++gk211q750iboMg2OYLjyYqcwbXn +-----END PRIVATE KEY----- diff --git a/tests/backend/mongodb-ssl/certs/test_mdb_mon_ssl_cert_and_key.pem b/tests/backend/mongodb-ssl/certs/test_mdb_mon_ssl_cert_and_key.pem new file mode 100644 index 00000000..92dc3c5e --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/test_mdb_mon_ssl_cert_and_key.pem @@ -0,0 +1,185 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 3 (0x3) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=ROOT-CA, CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + Validity + Not Before: Jun 14 12:44:48 2017 GMT + Not After : Jun 12 12:44:48 2027 GMT + Subject: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=MongoDB-Mon-Instance, CN=test-mdb-mon-ssl/emailAddress=dev@bigchaindb.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:ba:24:72:5f:73:ff:ea:19:08:30:c8:91:47:01: + 2c:ec:0e:b2:81:fa:84:01:72:38:d7:17:81:b3:87: + b0:ed:91:b2:d7:b1:7c:30:c6:d3:93:5f:39:04:e6: + fc:dd:b4:f6:c3:2a:d6:ce:ce:f1:6f:bf:32:05:92: + eb:90:66:0e:95:c5:ce:5c:3a:37:ac:cc:40:c5:ef: + 3e:53:b0:49:ed:ab:3c:9f:08:88:63:fa:a7:db:d4: + 6b:e9:55:0f:f3:f9:62:45:b9:a4:a2:3d:6a:60:b5: + ee:9c:e9:7c:cb:ef:06:0d:c5:92:21:4a:79:b4:f4: + 58:20:a6:8e:82:87:05:2c:c5:94:ab:6a:56:09:9e: + 64:bb:ab:29:86:04:a3:39:fd:4d:25:da:f7:6c:c9: + c2:c4:42:66:f4:f4:4e:91:dc:09:10:3d:6b:16:0b: + a4:be:c4:52:6e:4b:fa:8a:50:74:be:54:a8:46:e2: + 0d:53:db:9b:8d:6d:58:71:71:8e:f9:4f:ca:07:48: + c2:eb:fa:d1:42:0a:33:18:9e:14:0e:a8:6d:78:f6: + 8a:76:72:db:94:1e:56:cd:02:5d:7e:6d:6f:75:35: + cc:ca:c0:be:57:2c:6e:96:7f:79:51:84:ae:95:a5: + 3d:da:bd:01:8f:6a:de:8b:d0:dc:f8:61:27:f2:a6: + cf:0d:40:89:d0:2e:db:da:e5:ad:c5:9d:95:5f:22: + a4:52:70:33:4f:df:13:8f:96:a0:9b:21:aa:15:20: + 39:c6:16:65:98:b8:97:2a:cc:47:e0:79:41:b4:a2: + 96:32:68:65:f6:8c:1b:ce:c8:38:9c:75:2c:1a:87: + d9:87:9b:f8:b7:ff:2c:15:e2:0a:12:7b:a0:ef:3a: + 39:7e:32:3d:f9:42:d7:5f:08:38:93:4e:f1:41:a0: + c2:55:30:55:40:aa:bd:68:8e:69:c3:33:6f:50:0d: + ef:44:80:da:e5:01:61:d3:71:41:de:03:ab:42:85: + a0:80:1d:2a:b7:88:15:7b:ee:43:64:20:57:f7:25: + b2:6c:47:a6:5b:15:58:13:67:47:a3:07:17:20:51: + d2:8d:7b:71:91:39:5a:1b:a2:a0:0f:15:73:84:74: + 1f:49:b8:64:4b:3d:86:99:85:a5:e3:05:63:37:c3: + bf:e7:c2:4c:fc:ff:bc:9a:8f:43:43:52:a7:cb:b8: + 91:e7:0a:10:02:ac:74:3a:8b:06:34:50:d2:a3:12: + 81:d2:7b:46:bb:ad:2f:9e:ed:9e:0d:8d:27:64:99: + e5:e2:a2:56:cb:ad:3f:48:24:9e:11:32:bf:70:44: + e2:a5:bf:39:86:5b:ed:5d:75:c2:53:40:62:da:2d: + f8:f2:1d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + X509v3 Subject Key Identifier: + 31:63:2C:98:2F:9F:6C:44:82:A9:B8:D3:06:15:95:84:D9:52:98:71 + X509v3 Authority Key Identifier: + keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98 + DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + serial:E8:06:B2:C9:2A:9C:2E:FC + + X509v3 Extended Key Usage: + TLS Web Client Authentication + X509v3 Key Usage: + Digital Signature + Signature Algorithm: sha256WithRSAEncryption + 1e:16:02:5b:35:f6:36:0a:54:bc:48:11:51:39:a1:b1:e5:39: + bd:26:73:bc:37:22:95:87:3f:ed:e6:e1:00:fa:e2:a7:2d:ef: + 9d:25:ee:b0:7c:c2:e5:8e:9d:ff:24:51:ee:a2:cb:2c:b9:0a: + 38:07:94:8c:12:21:e1:1f:83:5f:4d:92:a8:b1:ff:53:90:97: + 30:2d:06:d6:84:79:27:6c:34:dc:19:6e:af:dd:80:a7:66:3b: + 0d:c5:c2:0d:7a:ce:b6:12:c2:9e:6f:02:0d:d0:41:c1:7d:75: + aa:07:46:50:e0:06:22:a8:d7:d2:45:dc:d3:c1:20:01:61:c6: + 07:13:74:b3:9e:de:88:1a:75:1d:8d:0a:3f:fd:0a:56:07:92: + d6:ce:37:f5:e6:ad:d0:64:33:77:36:dd:76:06:e1:20:00:64: + 88:d0:ca:71:f2:65:7c:26:ce:2c:55:07:50:36:d9:2b:b2:80: + fa:d1:4b:fc:31:89:d9:3e:c6:50:a8:ce:cf:df:d2:54:53:e7: + 80:ab:e6:4e:66:e8:91:70:55:95:80:94:74:60:f5:e8:ff:69: + 65:c0:41:17:af:1e:8a:50:a7:4e:f3:c1:76:42:7a:62:22:0a: + 51:33:06:57:bc:6f:7f:f6:5a:9d:4f:cd:2b:21:65:63:d8:ab: + 5b:38:8f:8c:f6:37:50:ca:32:5c:9a:3a:1b:a1:db:9f:fa:10: + 4e:35:54:9c:24:42:8a:33:58:a5:3e:b4:a4:67:4a:d2:b1:8d: + 99:d5:4d:1f:f7:d9:c6:ee:60:54:7f:bc:57:2f:0a:b9:ce:04: + 96:0f:0d:9c:22:39:a2:4e:e3:c7:3d:df:9c:09:af:45:62:57: + 1e:25:67:b1:4e:e9:15:88:c5:b3:2d:88:c8:60:8e:5e:b5:28: + 49:77:63:6f:0f:9d:d2:06:94:b4:b3:d9:92:2a:32:7f:45:c8: + 32:69:12:7d:8d:47:52:5b:3e:7f:f0:bc:80:11:56:08:97:7a: + eb:fb:0d:69:5d:88:b9:bb:27:7f:de:2e:a9:63:c5:89:56:88: + ce:2f:47:f0:1f:bc:3a:60:f4:19:8a:39:82:11:51:99:7a:8d: + 1f:11:53:2f:f4:43:48:08:c9:1e:a5:3b:ed:f0:8e:cc:d6:1e: + 20:e9:2a:4d:c3:cc:3a:6c:63:29:a7:1b:c3:63:13:19:d5:82: + 61:b9:83:39:e8:60:d9:06:e4:cc:55:5e:93:70:80:97:58:f4: + 24:1f:2b:6f:e6:ff:67:f1:76:2a:b3:db:6b:1a:22:42:a0:85: + 44:de:1a:9a:9e:b5:d2:ca:95:11:2d:ba:57:4e:2e:79:67:10: + c8:ee:aa:67:61:ca:8f:25 +-----BEGIN CERTIFICATE----- +MIIG4zCCBMugAwIBAgIBAzANBgkqhkiG9w0BAQsFADCBmzELMAkGA1UEBhMCREUx +DzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdj +aGFpbkRCIEdtYkgxEDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5m +cmEgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tMB4X +DTE3MDYxNDEyNDQ0OFoXDTI3MDYxMjEyNDQ0OFowgaYxCzAJBgNVBAYTAkRFMQ8w +DQYDVQQIDAZCZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEYMBYGA1UECgwPQmlnY2hh +aW5EQiBHbWJIMR0wGwYDVQQLDBRNb25nb0RCLU1vbi1JbnN0YW5jZTEZMBcGA1UE +AwwQdGVzdC1tZGItbW9uLXNzbDEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWlu +ZGIuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuiRyX3P/6hkI +MMiRRwEs7A6ygfqEAXI41xeBs4ew7ZGy17F8MMbTk185BOb83bT2wyrWzs7xb78y +BZLrkGYOlcXOXDo3rMxAxe8+U7BJ7as8nwiIY/qn29Rr6VUP8/liRbmkoj1qYLXu +nOl8y+8GDcWSIUp5tPRYIKaOgocFLMWUq2pWCZ5ku6sphgSjOf1NJdr3bMnCxEJm +9PROkdwJED1rFgukvsRSbkv6ilB0vlSoRuINU9ubjW1YcXGO+U/KB0jC6/rRQgoz +GJ4UDqhtePaKdnLblB5WzQJdfm1vdTXMysC+Vyxuln95UYSulaU92r0Bj2rei9Dc ++GEn8qbPDUCJ0C7b2uWtxZ2VXyKkUnAzT98Tj5agmyGqFSA5xhZlmLiXKsxH4HlB +tKKWMmhl9owbzsg4nHUsGofZh5v4t/8sFeIKEnug7zo5fjI9+ULXXwg4k07xQaDC +VTBVQKq9aI5pwzNvUA3vRIDa5QFh03FB3gOrQoWggB0qt4gVe+5DZCBX9yWybEem +WxVYE2dHowcXIFHSjXtxkTlaG6KgDxVzhHQfSbhkSz2GmYWl4wVjN8O/58JM/P+8 +mo9DQ1Kny7iR5woQAqx0OosGNFDSoxKB0ntGu60vnu2eDY0nZJnl4qJWy60/SCSe +ETK/cETipb85hlvtXXXCU0Bi2i348h0CAwEAAaOCASMwggEfMAkGA1UdEwQCMAAw +HQYDVR0OBBYEFDFjLJgvn2xEgqm40wYVlYTZUphxMIHQBgNVHSMEgcgwgcWAFGnH +hYBk5pBA6TBoiCPSTbvs35iYoYGhpIGeMIGbMQswCQYDVQQGEwJERTEPMA0GA1UE +CAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xGDAWBgNVBAoMD0JpZ2NoYWluREIg +R21iSDEQMA4GA1UECwwHUk9PVC1DQTEbMBkGA1UEAwwSVGVzdCBJbmZyYSBSb290 +IENBMSEwHwYJKoZIhvcNAQkBFhJkZXZAYmlnY2hhaW5kYi5jb22CCQDoBrLJKpwu +/DATBgNVHSUEDDAKBggrBgEFBQcDAjALBgNVHQ8EBAMCB4AwDQYJKoZIhvcNAQEL +BQADggIBAB4WAls19jYKVLxIEVE5obHlOb0mc7w3IpWHP+3m4QD64qct750l7rB8 +wuWOnf8kUe6iyyy5CjgHlIwSIeEfg19Nkqix/1OQlzAtBtaEeSdsNNwZbq/dgKdm +Ow3Fwg16zrYSwp5vAg3QQcF9daoHRlDgBiKo19JF3NPBIAFhxgcTdLOe3ogadR2N +Cj/9ClYHktbON/XmrdBkM3c23XYG4SAAZIjQynHyZXwmzixVB1A22SuygPrRS/wx +idk+xlCozs/f0lRT54Cr5k5m6JFwVZWAlHRg9ej/aWXAQRevHopQp07zwXZCemIi +ClEzBle8b3/2Wp1PzSshZWPYq1s4j4z2N1DKMlyaOhuh25/6EE41VJwkQoozWKU+ +tKRnStKxjZnVTR/32cbuYFR/vFcvCrnOBJYPDZwiOaJO48c935wJr0ViVx4lZ7FO +6RWIxbMtiMhgjl61KEl3Y28PndIGlLSz2ZIqMn9FyDJpEn2NR1JbPn/wvIARVgiX +euv7DWldiLm7J3/eLqljxYlWiM4vR/AfvDpg9BmKOYIRUZl6jR8RUy/0Q0gIyR6l +O+3wjszWHiDpKk3DzDpsYymnG8NjExnVgmG5gznoYNkG5MxVXpNwgJdY9CQfK2/m +/2fxdiqz22saIkKghUTeGpqetdLKlREtuldOLnlnEMjuqmdhyo8l +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC6JHJfc//qGQgw +yJFHASzsDrKB+oQBcjjXF4Gzh7DtkbLXsXwwxtOTXzkE5vzdtPbDKtbOzvFvvzIF +kuuQZg6Vxc5cOjeszEDF7z5TsEntqzyfCIhj+qfb1GvpVQ/z+WJFuaSiPWpgte6c +6XzL7wYNxZIhSnm09Fggpo6ChwUsxZSralYJnmS7qymGBKM5/U0l2vdsycLEQmb0 +9E6R3AkQPWsWC6S+xFJuS/qKUHS+VKhG4g1T25uNbVhxcY75T8oHSMLr+tFCCjMY +nhQOqG149op2ctuUHlbNAl1+bW91NczKwL5XLG6Wf3lRhK6VpT3avQGPat6L0Nz4 +YSfyps8NQInQLtva5a3FnZVfIqRScDNP3xOPlqCbIaoVIDnGFmWYuJcqzEfgeUG0 +opYyaGX2jBvOyDicdSwah9mHm/i3/ywV4goSe6DvOjl+Mj35QtdfCDiTTvFBoMJV +MFVAqr1ojmnDM29QDe9EgNrlAWHTcUHeA6tChaCAHSq3iBV77kNkIFf3JbJsR6Zb +FVgTZ0ejBxcgUdKNe3GROVoboqAPFXOEdB9JuGRLPYaZhaXjBWM3w7/nwkz8/7ya +j0NDUqfLuJHnChACrHQ6iwY0UNKjEoHSe0a7rS+e7Z4NjSdkmeXiolbLrT9IJJ4R +Mr9wROKlvzmGW+1ddcJTQGLaLfjyHQIDAQABAoICAEU5ysNPC6zPFGm4HbdPa5p6 +uM54EWHMTfFIT7cpzpX7qoxm+G3Xc9YKAtWc5gu6Ak+A/hQ5iDbr3zmNc0fnfKMc +xmssR4pfB/PUztQm6seV+FyYusib7pNlw6AqP12XsIGH0f94YbiH0K7QctYRFapH +gNoarXqrqJ6z8qn+duE4tdquGENJgeL9e9rYnK+zUV6zuXLe8i01GL8eoJ3EPCar +AMlZGTLNJm7YmNuZomEqcM6zwQsf0BDfypWeCvMTRmpBGCTUycAKtQgBuindX6Et +5z635ouGYPerWoPrRRlNIdWBqwgWgSY9UCwN8TO8GW3g/tSMhSlwSs2grE+k07lR +6d05taL8c/thZDpHBePVhKvNvJfI8ef0lqCCPc/uFnmWkY3pg2HB3Z605YqPhddL +tTqaDZVL2oc9qrnyjz/6cFOa9Mh3cdt8L1+j0hMDWxYOcYS7IJi35V25B8wDPVZj +LXCINNQtTio7hwc+jyXhPVSERfeISlU1Ki5ZomWLzMtkdDzBz8By++aEudKw+cV/ +OjLa07SLdn05j52rqYhzaZEbeXoY1L2N+V9c86iukjr8vLHStzgmzpWDYcsnHu7w +Ojy4iRtAvI2a+PSq0tdqfhFKCA5zevLcCxUUhSDxWIa81TYEE+MfqQYtEASB1/Ze +Nt6IKu9zpLXA1WFNxLktAoIBAQDvU/K+eznf/ro46WvpKqXyU0YkEoMNHDQ7ljrG +QLkDo9WfN08aKpi6tjdgKZ97yqJae16h6WJj6aDyKL+kpkhitQ8QPhvpg4Efiupz +15gqbLzlIqrmdfGH/X2H/C3i6WDdrJWMA7ERitK4TKGoKzIH+YHA3kjEwvVnAh/U +TKov47H40UP8qTTmortTC9uO+3TmbpSkDU7iGhTKd1RYcLnKj9jmI+//BZ0OYe6U +QzrCKD27YIsH0QD9pWv/Wn3Xy2yH7EnWxXzxq6Wd767JHIloHVYICrnMf/rSA/sV +8g/JjLuLCUf2U0DxauX7wcnC7XtOJ0u5PBw1lzRYA/vfNgVPAoIBAQDHHAO9K2E7 +5Gy2zVVdqxgK4NqY6pOgaEu7T95VaTchShQ3dGs5b/s8GU+gjZ+XOF4VEqjazGR5 +xUXqlF+XLp0YI7vn9yNoUYRvHtbun8hBaQ2yc4iWzxGeAQVCYr8e0w6gs7pyFIh5 +xKZzaYZFKIJhTD/NApuul0IYnRSnKXhODp4x/dElpgectkLUHlME9rrXjBdKQVFN +x50JBnopA0/npGxtuDzztSP7qq1rFp3HHMChvzM5oLoOj5b80OmaI52vCGY6Udhy +HMNbMzMiEHgekNramFEe0fZxuwUkF6+O+FIb+jPy0EP04an4IDNhjLet7PHN62qW +CEF1UIz/w87TAoIBABnY8TCNUnTUp/wZiaQHDSEcyUiAD1NNJn8A/JwIxXKVMS8e +5BYpX5FauKRUGnteKSaoiFHoSM4Nn2pH6Fq58rtmXpgcfRs9Lqbfc+7K7A447DW7 +BsYZGtrbD8GuBK6rEeEfEI+snmUMnzF0ZUkqUR73XYIc/7Lwc1yKqDFfjknZx5Dd +2P/irmnyTVTsxOuuULPKYZdOMKTOuEwdkyhyFD7CRmSkoPjj/FBfV0r/78qkWfZw +uNNBSWAbi4xqk4jI2ZHcfSUK8zGOnjZuemwR/u0VrgL0VZi2gbpI38dA+1+DKYYc +nH8IAs2QBAKqrUW2LEkGiXEaAtnu2KJg28UBqpUCggEABSMw6YGkCaKUN4dGy5rt +jOJOYaGz23C1e4jNCNElLgO2T9P+LEY6akuQ5WiiFInMy6hmmParJQU1D59yc/ks +7oGFiK+0xy1LSH64NwICbcWjJ9aZUKLZJoWKODNKESaK+xSIHAdxmgq77MBtSX2J +F6+a+dQ2ZiPa/b2X2dRVGVaBOHL/IYSOL7n4MXby127yVTt1ImJbEbGz2JbFMie2 +uRhh/9bAI92ppwW5Ycj8mzWftsyzKqp+AoAr+iv9yw9eMzT5Rkn0VsVtOP4yNI/O +OaOtMfS6Kxxpyndz44GE8yBvJe+nxX0gM8Ja218hVEQIPUGe35xSbLqbzcYdTUAl +JwKCAQBWx+ej+03fXOPHN8K2IA1a+8wEwjxsjbpGtHd/nyDVkjdqVHW+Jkmyqkne +gFbDvq2ZHQJjDYB9o/ZnwXpEXfweSz2XsdiNBMQvIUMOGRmI9yJJ1Ez1gfhlJYpW +Im+/dWQCTw0o/8cHPI32ic3iBZBuemGCxccF89bmX+GPzK/aT8F+5uG/pHjHXNU6 +S/jKxZIwDM9yW4vIRdOAAi1eya0f+VHE70ORpUXfkUV5kVQR7pLEWDFDKIF/zBGv +vRPdZOO5GOQb0kQ1eHyHDPUOimT1GlF+e9OLvy6dVDaqKRBEiVHjxfWGUnpH+Qo5 +zlmQnC98+HETU0YHNxAAMsRPvxXi +-----END PRIVATE KEY----- diff --git a/tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem b/tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem new file mode 100644 index 00000000..e9924e4b --- /dev/null +++ b/tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem @@ -0,0 +1,188 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1 (0x1) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=ROOT-CA, CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + Validity + Not Before: Jun 13 14:40:55 2017 GMT + Not After : Jun 11 14:40:55 2027 GMT + Subject: C=DE, ST=Berlin, L=Berlin, O=BigchainDB GmbH, OU=MongoDB-Instance, CN=test-mdb-ssl/emailAddress=dev@bigchaindb.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:bb:37:f1:24:2f:83:95:2c:bf:47:a4:a0:2c:9f: + e1:bf:fb:70:f6:9b:04:a0:e8:3b:81:6f:ca:a6:22: + 6d:63:75:48:bc:fa:57:29:16:f7:2f:09:52:6c:c5: + 52:20:74:9b:27:99:5b:51:85:ea:ef:c9:5f:78:c9: + 29:b8:cc:a4:a8:46:c2:a3:64:41:84:92:36:c7:a8: + 82:e2:b6:5e:67:08:d6:bb:3d:36:06:31:10:53:7a: + 58:66:66:89:4d:46:d3:f6:3e:38:d4:84:d9:6c:c4: + 95:68:9e:66:b7:47:c6:63:dc:35:1c:46:3d:8e:c6: + 96:13:aa:65:53:3d:75:84:76:af:85:d2:6f:97:78: + 0f:d2:8a:c0:cb:3f:86:30:09:0f:bf:ae:30:cb:26: + 05:61:0c:ad:07:5e:33:cc:82:bb:46:49:86:a5:6e: + 07:1b:78:b2:71:7b:01:ee:00:97:ac:81:49:89:df: + 73:ab:d7:78:8e:a3:c5:76:5c:e9:fd:48:5f:a6:45: + b8:97:13:6a:55:05:5e:e2:00:46:27:67:93:06:fb: + 91:83:69:c7:5e:12:49:ce:39:b9:ca:69:9f:0d:ff: + 6a:79:b6:6b:6f:81:6c:51:e2:d0:01:9b:f8:7b:25: + 3e:09:4d:00:aa:89:7f:00:e3:fb:5c:35:19:8d:c2: + 11:21:87:1e:7a:11:bd:88:b2:de:ea:bf:a0:8b:fb: + 0c:c3:6d:e2:ce:1f:6c:5b:5f:4a:05:da:ff:f1:fd: + 4a:f5:de:5b:d8:93:ca:17:6b:dc:80:3e:91:6e:97: + 43:db:59:f5:80:c1:0a:54:32:cd:c3:be:87:e9:93: + 14:ae:c7:29:01:90:e8:11:32:59:1a:73:bf:42:0d: + c6:82:2c:89:2e:54:67:c8:2a:c3:3b:ce:e6:c1:09: + 48:ad:d3:a3:e1:80:f8:df:6e:4c:78:72:76:db:4e: + d9:b9:fa:5b:7c:85:73:64:c9:23:94:ae:5e:63:68: + 7f:1e:63:d3:78:85:47:0b:ec:52:1e:02:cf:7b:9d: + 96:9c:63:4f:46:f7:79:1e:90:d3:21:18:85:26:17: + f1:51:18:d6:87:1c:9b:50:0e:70:6b:f1:08:41:b6: + 3c:fc:fb:d7:1f:f1:bd:2d:3e:77:b6:66:92:0a:81: + 01:0f:4a:68:68:69:5c:0f:38:b3:46:68:1b:55:99: + 67:29:d1:03:f2:a5:5b:f1:8e:53:ed:c7:cc:79:38: + 9c:8d:a6:78:f4:3e:23:28:ee:a0:d3:55:0f:c3:f0: + 64:c1:fb:e5:85:71:b2:1e:44:d3:1d:78:87:77:76: + e2:6f:5f:54:64:14:b4:2f:14:0b:a5:3d:98:fb:68: + 99:1f:23 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + X509v3 Subject Key Identifier: + 0B:53:E2:76:40:AD:73:C4:12:6C:85:CF:36:5B:5F:FB:6E:E8:03:A7 + X509v3 Authority Key Identifier: + keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98 + DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com + serial:E8:06:B2:C9:2A:9C:2E:FC + + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost, DNS:test-mdb-ssl + Signature Algorithm: sha256WithRSAEncryption + 4c:14:3e:6b:af:f8:e8:69:11:2e:13:12:b7:9b:91:c7:68:01: + 26:00:b8:c8:35:8b:fe:d2:bb:ab:43:d1:7a:8e:24:b2:08:dd: + 1a:77:91:f0:68:35:42:56:ba:fe:26:3e:91:e2:8c:c1:01:e2: + 65:f7:3b:12:ba:7f:1e:8a:8e:5b:a1:c8:28:8c:16:b8:72:03: + 31:d6:6c:2c:ac:80:6e:7d:52:24:2e:4d:0b:e6:90:d1:7d:18: + 3f:ea:9f:7f:85:39:86:77:3b:19:3d:ba:b0:57:10:16:25:fc: + d6:be:17:7b:c4:92:0e:c7:18:3c:69:48:e0:72:2c:3e:42:2f: + 0b:70:02:a8:c1:04:2b:d8:00:72:b4:67:35:d9:79:3f:98:71: + 55:92:e6:fa:51:2c:42:2f:71:c6:4f:98:7f:d8:2c:7c:12:70: + 97:ad:cd:92:0a:66:80:2a:ec:ac:e6:9b:3a:0b:27:ca:e0:cc: + 9f:b4:07:f4:fa:f7:60:17:39:f1:46:46:eb:e5:78:2a:84:b2: + 78:87:ce:73:ad:20:8b:50:8c:d5:c5:cd:4c:b1:96:be:64:24: + e0:a9:81:c3:01:51:a2:b1:50:22:15:97:5a:e6:49:f3:1d:f1: + 72:3e:8f:0d:87:e9:05:c1:92:8e:4b:db:1b:e2:b8:3c:b4:13: + dd:3d:ce:4d:f8:1e:8e:73:ae:5a:36:ba:be:dd:11:7c:b8:b1: + ef:d2:94:84:a7:c4:0d:96:0f:e1:46:46:bb:7d:51:a9:61:13: + 98:47:b2:68:ad:85:0c:f9:32:0a:76:49:20:6f:34:72:ca:06: + fd:05:6f:16:5d:10:67:3c:50:06:f3:c2:bd:58:c8:f4:b3:96: + dc:28:26:62:e7:30:a0:0c:40:f6:7c:50:42:21:c8:e1:73:64: + 1b:9c:76:3e:78:1d:ea:54:fc:61:6c:3e:27:59:cb:c4:dd:9f: + 94:bf:b5:13:87:79:ff:28:1a:9b:7f:2f:1f:9b:22:1d:30:f6: + c9:5f:53:6f:1a:88:38:b0:44:71:79:da:a6:0f:2b:e7:42:71: + 9c:3a:20:7f:6b:bb:93:71:b4:6c:9e:2a:b7:fa:57:cf:81:bd: + 21:eb:0a:db:83:07:ac:fd:79:cb:ef:c7:fd:cd:ef:22:7f:67: + 71:7c:e2:5c:40:8e:f1:06:c3:75:67:6a:70:f4:80:b3:ad:c6: + 89:31:9b:cf:8c:ae:6e:85:fd:51:c6:40:34:a2:0c:63:55:84: + e1:a9:10:00:48:1a:64:95:80:45:09:d2:1d:3f:e2:6c:e1:e8: + 92:28:5f:da:a1:69:10:74:03:1f:f1:43:c4:43:fb:01:80:7e: + fb:0b:2c:62:ad:e5:f2:61 +-----BEGIN CERTIFICATE----- +MIIHCTCCBPGgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBmzELMAkGA1UEBhMCREUx +DzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdj +aGFpbkRCIEdtYkgxEDAOBgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5m +cmEgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tMB4X +DTE3MDYxMzE0NDA1NVoXDTI3MDYxMTE0NDA1NVowgZ4xCzAJBgNVBAYTAkRFMQ8w +DQYDVQQIDAZCZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEYMBYGA1UECgwPQmlnY2hh +aW5EQiBHbWJIMRkwFwYDVQQLDBBNb25nb0RCLUluc3RhbmNlMRUwEwYDVQQDDAx0 +ZXN0LW1kYi1zc2wxITAfBgkqhkiG9w0BCQEWEmRldkBiaWdjaGFpbmRiLmNvbTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALs38SQvg5Usv0ekoCyf4b/7 +cPabBKDoO4FvyqYibWN1SLz6VykW9y8JUmzFUiB0myeZW1GF6u/JX3jJKbjMpKhG +wqNkQYSSNseoguK2XmcI1rs9NgYxEFN6WGZmiU1G0/Y+ONSE2WzElWieZrdHxmPc +NRxGPY7GlhOqZVM9dYR2r4XSb5d4D9KKwMs/hjAJD7+uMMsmBWEMrQdeM8yCu0ZJ +hqVuBxt4snF7Ae4Al6yBSYnfc6vXeI6jxXZc6f1IX6ZFuJcTalUFXuIARidnkwb7 +kYNpx14SSc45ucppnw3/anm2a2+BbFHi0AGb+HslPglNAKqJfwDj+1w1GY3CESGH +HnoRvYiy3uq/oIv7DMNt4s4fbFtfSgXa//H9SvXeW9iTyhdr3IA+kW6XQ9tZ9YDB +ClQyzcO+h+mTFK7HKQGQ6BEyWRpzv0INxoIsiS5UZ8gqwzvO5sEJSK3To+GA+N9u +THhydttO2bn6W3yFc2TJI5SuXmNofx5j03iFRwvsUh4Cz3udlpxjT0b3eR6Q0yEY +hSYX8VEY1occm1AOcGvxCEG2PPz71x/xvS0+d7ZmkgqBAQ9KaGhpXA84s0ZoG1WZ +ZynRA/KlW/GOU+3HzHk4nI2mePQ+IyjuoNNVD8PwZMH75YVxsh5E0x14h3d24m9f +VGQUtC8UC6U9mPtomR8jAgMBAAGjggFRMIIBTTAJBgNVHRMEAjAAMB0GA1UdDgQW +BBQLU+J2QK1zxBJshc82W1/7bugDpzCB0AYDVR0jBIHIMIHFgBRpx4WAZOaQQOkw +aIgj0k277N+YmKGBoaSBnjCBmzELMAkGA1UEBhMCREUxDzANBgNVBAgMBkJlcmxp +bjEPMA0GA1UEBwwGQmVybGluMRgwFgYDVQQKDA9CaWdjaGFpbkRCIEdtYkgxEDAO +BgNVBAsMB1JPT1QtQ0ExGzAZBgNVBAMMElRlc3QgSW5mcmEgUm9vdCBDQTEhMB8G +CSqGSIb3DQEJARYSZGV2QGJpZ2NoYWluZGIuY29tggkA6AayySqcLvwwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAsGA1UdDwQEAwIFoDAiBgNVHREEGzAZ +gglsb2NhbGhvc3SCDHRlc3QtbWRiLXNzbDANBgkqhkiG9w0BAQsFAAOCAgEATBQ+ +a6/46GkRLhMSt5uRx2gBJgC4yDWL/tK7q0PReo4ksgjdGneR8Gg1Qla6/iY+keKM +wQHiZfc7Erp/HoqOW6HIKIwWuHIDMdZsLKyAbn1SJC5NC+aQ0X0YP+qff4U5hnc7 +GT26sFcQFiX81r4Xe8SSDscYPGlI4HIsPkIvC3ACqMEEK9gAcrRnNdl5P5hxVZLm ++lEsQi9xxk+Yf9gsfBJwl63NkgpmgCrsrOabOgsnyuDMn7QH9Pr3YBc58UZG6+V4 +KoSyeIfOc60gi1CM1cXNTLGWvmQk4KmBwwFRorFQIhWXWuZJ8x3xcj6PDYfpBcGS +jkvbG+K4PLQT3T3OTfgejnOuWja6vt0RfLix79KUhKfEDZYP4UZGu31RqWETmEey +aK2FDPkyCnZJIG80csoG/QVvFl0QZzxQBvPCvVjI9LOW3CgmYucwoAxA9nxQQiHI +4XNkG5x2Pngd6lT8YWw+J1nLxN2flL+1E4d5/ygam38vH5siHTD2yV9TbxqIOLBE +cXnapg8r50JxnDogf2u7k3G0bJ4qt/pXz4G9IesK24MHrP15y+/H/c3vIn9ncXzi +XECO8QbDdWdqcPSAs63GiTGbz4yuboX9UcZANKIMY1WE4akQAEgaZJWARQnSHT/i +bOHokihf2qFpEHQDH/FDxEP7AYB++wssYq3l8mE= +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC7N/EkL4OVLL9H +pKAsn+G/+3D2mwSg6DuBb8qmIm1jdUi8+lcpFvcvCVJsxVIgdJsnmVtRhervyV94 +ySm4zKSoRsKjZEGEkjbHqILitl5nCNa7PTYGMRBTelhmZolNRtP2PjjUhNlsxJVo +nma3R8Zj3DUcRj2OxpYTqmVTPXWEdq+F0m+XeA/SisDLP4YwCQ+/rjDLJgVhDK0H +XjPMgrtGSYalbgcbeLJxewHuAJesgUmJ33Or13iOo8V2XOn9SF+mRbiXE2pVBV7i +AEYnZ5MG+5GDacdeEknOObnKaZ8N/2p5tmtvgWxR4tABm/h7JT4JTQCqiX8A4/tc +NRmNwhEhhx56Eb2Ist7qv6CL+wzDbeLOH2xbX0oF2v/x/Ur13lvYk8oXa9yAPpFu +l0PbWfWAwQpUMs3DvofpkxSuxykBkOgRMlkac79CDcaCLIkuVGfIKsM7zubBCUit +06PhgPjfbkx4cnbbTtm5+lt8hXNkySOUrl5jaH8eY9N4hUcL7FIeAs97nZacY09G +93kekNMhGIUmF/FRGNaHHJtQDnBr8QhBtjz8+9cf8b0tPne2ZpIKgQEPSmhoaVwP +OLNGaBtVmWcp0QPypVvxjlPtx8x5OJyNpnj0PiMo7qDTVQ/D8GTB++WFcbIeRNMd +eId3duJvX1RkFLQvFAulPZj7aJkfIwIDAQABAoICAAI7J2+D9XB4qQrkhlghy6Hr +ECyQKlz0h4kCWQrjzCPsSOKfpRxDPszbspTDQThLy3GMXU86ZlNXNgENfKbMIYYz +2avyzOkUiPyWIIEtJTDbJRv0HcmzGfu0sIr/29EE8A+2LB00PBcUvKcThCvY+h9E +h4l/uMoTycQQOsbkK5tZgrv4hwXqE14x8xw49JNr+DkTjVdOa6/9Y657y+g7AppS +0/zys7j7Fj1N9vbsIOKYY9T1yb4ZgkFI6FiBwPQYwbMMj6eVPqYmuzu8PxnHHTRv +GQvU3eBcHW8Mtw8XdSLIkW9D4kHg6/aJ2mq6fhqRlgysUz61BsNu9BbkNxq+Xk/f +wU+Slo0UcnuU49icYo8J2yPULUgQaP4VdJc7tb9kMpklTUG7YaD5oqKX+I/jWS9U +Mta1h3GoK2zK9pj2B5YyCoUsM8o4EEb3hZ8FYf0DOuVd9XcWxLUDRGQoTWJptcSW +o5OHaP2tgcUtq6siiqPXIQQr1Yji/geSFTP/hWOZC4AFgwa9XvboD3PcKsG2gIc8 +I/HLF5tCpUXIAW4wluXQv1MveX9xwaez+PGLTIFCHidgOBmC5jZEijG9PUgRk1D1 +e8CUT9Vc3zxS/9S+RfSwXMEBX9367edoiNw1MuXvR1nwCFYKthONx8ww1psEoISF +9ZMbdac3hwqeqSW34ZoJAoIBAQDfOF6D1vWieXwQ7W3KpAxyHlNVN8tus3w+mDBc +unv3Nsn5gmDYHc/Iu5/Tyk2zgoEsfN1LZgzjmSBCUgJOz09+8yrdpsTI4awXAwoa +kaczRcutjAG5vHhHXi5+iwqAaPSTBYcAD2d5SGtxpWBJ2qywT3DMWFT25bhRj3Bp +mc3Fe6oxbz2QV2Is2SnnCnPc/Ok7cABwaP1mc3PsIZzswuJjq3JWsMAlBMZueQW1 +CxI639j2memt7ZSWU0UDLg4RAPOtcSt/NykbZgAlfQoLiKWv2hhdvip+oPsdOiDd +tYmeoseRd/7Ef8EdSwvSDh1jTugu6PG0NeK0Gfxs5Ipb1eBvAoIBAQDWtiTD8dP3 +TNbLu3nkopgJJe9gn1dVhvhBY0T4ar9pR1yl8s5L13vtmrs5wQ5/vyZ2A39OR4fH +5PcT/gcPm4GuE1BL29NBz88Qdz6033hOuY4SvZpf94yeDhY2w2Gfzm0uewGDR0z+ +EOsSGnnpEYvr+Bkdcvj3CP0ksiQPa/mAnmEhVvthlavyKhfxvwNyh07qym8Q8r3R +BiwChB/zGo6V1sSfPexy0I9kPhXByYyxoADn/lgXKXtnXpP7fuQleztPC+84bNpa +PFo5QnzG2kq+XwTP0Z3kQiZ2hEg+nGWHa7MpG+Ic7sDZE2MPbpUAY59J9HBmJOtK +jaMDQpN1OJ6NAoIBAQCR5ZMJEu/bsRGhMr5nC1BDKoUHFv+1LEUR2IELqqupyjq7 +BTdtKhcMOW3ejMS6hpsNS9OMG3hjLkuCbX2qqx9nd2JWG1FVhwLmy4rIENk6zuyw +Uukxz/jBlNJIzXIgufjZO2ygYnEcOXRYhGHqA66JUnMEbNujwEP3swwjSW7QG+MC +ipWviR3W+F3F7/QIxZlfyvXOgyDU3LMw7EtYCIzV3lJOTRpZV8Smoh1T1pekuXhi +vfiZQMa0dlqOGIUDIh8jBuwKKzCIAuQnnJF7NmYtCRyR8OBvgaJ/2OVwY94+PGC6 +OLbhEitTjSB9u9IDfaiOOJz6T5oMY0u0lK/GFIWrAoIBAQCkpr8g6ATlYy5gw57g +/vLaKRYdoXOmqb8c3ZRZb/rjMA9TTxR2QoPNnNewnWSSADLtUxdAH0h3uRTkZuxh +Qp46sKLl2Why0np2pQsYRzqKHG9f3bhRmZNi7WiJjGED3JgLidaKJpZbSvwJQPZ2 +DbegN/yCgdV8E4+UoWtXzDRkNpBDseFKXy2hojgEcbeiDzMsiBqOA6geb742G74o +fhgTvcPAXgtBrMAZXibvGbFj0VCAl6JT+MKibXvL4+3u9EZyArPrMEZt5lRGDr+C ++olQ3arh91w+W6AFSMHiCasuIyAcCT7ncwu+K8jOX72xs7PGUcYT/oHVBTKZ4GcW +AFpNAoIBAB/F8olQT10+QCZciP1ZuRpdEbEheWAF9ng3E7r7wAQiWNoLHj10Nu3H +byHPMmXNsw2VwaHqrAb5nTl5hfpk/ph7SidxMyNbtjCAlYm7XCiSMLLj0v5sFyCc +wgZqXzttQ6NRKt90JeIvHiazP7SVrmCyU/RhMlufiyX5zvivOgdtQA+0ISFDGdcX +MgwLTdFci9RaS5WDYsvokcc/5xEP4bZnxgEbmVq19mIC8QrCdH7qh/KP9Psb2tIt +OGwGA/YSMeQysKjUqAb/mcaMYEf2GUwWY6mZfRfLbeTT8yeDKFvAx5IUIz5tkFD/ +nMEENsuszPYolIDIv8pISWC8yCsWn5w= +-----END PRIVATE KEY----- diff --git a/tests/backend/mongodb-ssl/test_ssl_connection.py b/tests/backend/mongodb-ssl/test_ssl_connection.py new file mode 100644 index 00000000..ed31d0de --- /dev/null +++ b/tests/backend/mongodb-ssl/test_ssl_connection.py @@ -0,0 +1,215 @@ +from unittest import mock + +import pytest +import pymongo +from pymongo import MongoClient +from pymongo.database import Database +from ssl import CERT_REQUIRED + + +pytestmark = pytest.mark.bdb_ssl + + +@pytest.fixture +def mock_ssl_cmd_line_opts(certs_dir): + return {'argv': [ + 'mongod', + '--dbpath=/data', + '--replSet=bigchain-rs', + '--sslMode=requireSSL', + '--sslAllowInvalidHostnames', + '--sslCAFile=' + certs_dir + '/ca.crt', + '--sslCRLFile=' + certs_dir + '/crl.pem', + '--sslPEMKeyFile=' + certs_dir + '/test_mdb_ssl_cert_and_key.pem', + '--sslPEMKeyPassword=""' + ], + 'ok': 1.0, + 'parsed': {'replication': {'replSet': 'bigchain-rs'}, + 'storage': {'dbPath': '/data'}} + } + + +@pytest.fixture +def mock_ssl_config_opts(certs_dir): + return {'argv': [ + 'mongod', + '--dbpath=/data', + '--replSet=bigchain-rs', + '--sslMode=requireSSL', + '--sslAllowInvalidHostnames', + '--sslCAFile=' + certs_dir + '/ca.crt', + '--sslCRLFile=' + certs_dir + '/crl.pem', + '--sslPEMKeyFile=' + certs_dir + '/test_mdb_ssl_cert_and_key.pem', + '--sslPEMKeyPassword=""' + ], + 'ok': 1.0, + 'parsed': {'replication': {'replSetName': 'bigchain-rs'}, + 'storage': {'dbPath': '/data'}} + } + + +@pytest.fixture +def mongodb_ssl_connection(certs_dir): + import bigchaindb + return MongoClient(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port'], + serverselectiontimeoutms=bigchaindb.config['database']['connection_timeout'], + ssl=bigchaindb.config['database']['ssl'], + ssl_ca_certs=bigchaindb.config['database']['ca_cert'], + ssl_certfile=bigchaindb.config['database']['certfile'], + ssl_keyfile=bigchaindb.config['database']['keyfile'], + ssl_pem_passphrase=bigchaindb.config['database']['keyfile_passphrase'], + ssl_crlfile=bigchaindb.config['database']['crlfile'], + ssl_cert_reqs=CERT_REQUIRED) + + +def test_ssl_get_connection_returns_the_correct_instance(db_host, db_port, certs_dir): + from bigchaindb.backend import connect + from bigchaindb.backend.connection import Connection + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + config = { + 'backend': 'mongodb', + 'host': db_host, + 'port': db_port, + 'name': 'test', + 'replicaset': 'bigchain-rs', + 'ssl': True, + 'ca_cert': certs_dir + '/ca.crt', + 'crlfile': certs_dir + '/crl.pem', + 'certfile': certs_dir + '/test_bdb_ssl.crt', + 'keyfile': certs_dir + '/test_bdb_ssl.key', + 'keyfile_passphrase': '' + } + + conn = connect(**config) + assert isinstance(conn, Connection) + assert isinstance(conn, MongoDBConnection) + assert conn.conn._topology_settings.replica_set_name == config['replicaset'] + + +@mock.patch('pymongo.database.Database.authenticate') +def test_ssl_connection_with_credentials(mock_authenticate): + import bigchaindb + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + conn = MongoDBConnection(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port'], + login='theplague', + password='secret', + ssl=bigchaindb.config['database']['ssl'], + ssl_ca_certs=bigchaindb.config['database']['ca_cert'], + ssl_certfile=bigchaindb.config['database']['certfile'], + ssl_keyfile=bigchaindb.config['database']['keyfile'], + ssl_pem_passphrase=bigchaindb.config['database']['keyfile_passphrase'], + ssl_crlfile=bigchaindb.config['database']['crlfile'], + ssl_cert_reqs=CERT_REQUIRED) + conn.connect() + assert mock_authenticate.call_count == 2 + + +def test_ssl_initialize_replica_set(mock_ssl_cmd_line_opts, certs_dir): + from bigchaindb.backend.mongodb.connection import initialize_replica_set + from bigchaindb.common.exceptions import ConfigurationError + + with mock.patch.object(Database, 'command') as mock_command: + mock_command.side_effect = [ + mock_ssl_cmd_line_opts, + None, + {'log': ['database writes are now permitted']}, + ] + + # check that it returns + assert initialize_replica_set('host', + 1337, + 1000, + 'dbname', + True, + None, + None, + certs_dir + '/ca.crt', + certs_dir + '/test_bdb_ssl.crt', + certs_dir + '/test_bdb_ssl.key', + '', + certs_dir + '/crl.pem') is None + + # test it raises OperationError if anything wrong + with mock.patch.object(Database, 'command') as mock_command: + mock_command.side_effect = [ + mock_ssl_cmd_line_opts, + pymongo.errors.OperationFailure(None, details={'codeName': ''}) + ] + + with pytest.raises(pymongo.errors.OperationFailure): + initialize_replica_set('host', + 1337, + 1000, + 'dbname', + True, + None, + None, + certs_dir + '/ca.crt', + certs_dir + '/test_bdb_ssl.crt', + certs_dir + '/test_bdb_ssl.key', + '', + certs_dir + '/crl.pem') is None + + # pass an explicit ssl=False so that pymongo throws a + # ConfigurationError + with pytest.raises(ConfigurationError): + initialize_replica_set('host', + 1337, + 1000, + 'dbname', + False, + None, + None, + certs_dir + '/ca.crt', + certs_dir + '/test_bdb_ssl.crt', + certs_dir + '/test_bdb_ssl.key', + '', + certs_dir + '/crl.pem') is None + + +def test_ssl_invalid_configuration(db_host, db_port, certs_dir): + from bigchaindb.backend import connect + from bigchaindb.common.exceptions import ConfigurationError + + config = { + 'backend': 'mongodb', + 'host': db_host, + 'port': db_port, + 'name': 'test', + 'replicaset': 'bigchain-rs', + 'ssl': False, + 'ca_cert': certs_dir + '/ca.crt', + 'crlfile': certs_dir + '/crl.pem', + 'certfile': certs_dir + '/test_bdb_ssl.crt', + 'keyfile': certs_dir + '/test_bdb_ssl.key', + 'keyfile_passphrase': '' + } + + with pytest.raises(ConfigurationError): + conn = connect(**config) + assert conn.conn._topology_settings.replica_set_name == config['replicaset'] + + +def test_ssl_connection_with_wrong_credentials(): + import bigchaindb + from bigchaindb.backend.mongodb.connection import MongoDBConnection + from bigchaindb.backend.exceptions import ConnectionError + + conn = MongoDBConnection(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port'], + login='my_login', + password='my_super_secret_password', + ssl=bigchaindb.config['database']['ssl'], + ssl_ca_certs=bigchaindb.config['database']['ca_cert'], + ssl_certfile=bigchaindb.config['database']['certfile'], + ssl_keyfile=bigchaindb.config['database']['keyfile'], + ssl_pem_passphrase=bigchaindb.config['database']['keyfile_passphrase'], + ssl_crlfile=bigchaindb.config['database']['crlfile'], + ssl_cert_reqs=CERT_REQUIRED) + + with pytest.raises(ConnectionError): + conn._connect() diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index 3edc31b1..27be214a 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -180,7 +180,8 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] # check that it returns - assert initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None) is None + assert initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None, + None, None, None, None, None) is None # test it raises OperationError if anything wrong with mock.patch.object(Database, 'command') as mock_command: @@ -190,4 +191,5 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] with pytest.raises(pymongo.errors.OperationFailure): - initialize_replica_set('host', 1337, 1000, 'dbname', False, None, None) + initialize_replica_set('host', 1337, 1000, 'dbname', False, None, + None, None, None, None, None, None) is None diff --git a/tests/common/conftest.py b/tests/common/conftest.py index e8c4f9c6..8dfabf30 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -1,3 +1,4 @@ +from base58 import b58decode import pytest @@ -11,8 +12,13 @@ USER3_PRIVATE_KEY = '4rNQFzWQbVwuTiDVxwuFMvLG5zd8AhrQKCtVovBvcYsB' USER3_PUBLIC_KEY = 'Gbrg7JtxdjedQRmr81ZZbh1BozS7fBW88ZyxNDy7WLNC' -CC_FULFILLMENT_URI = 'cf:0:' -CC_CONDITION_URI = 'cc:0:3:47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU:0' +CC_FULFILLMENT_URI = ( + 'pGSAINdamAGCsQq31Uv-08lkBzoO4XLz2qYjJa8CGmj3B1EagUDlVkMAw2CscpCG4syAboKKh' + 'Id_Hrjl2XTYc-BlIkkBVV-4ghWQozusxh45cBz5tGvSW_XwWVu-JGVRQUOOehAL' +) +CC_CONDITION_URI = ('ni:///sha-256;' + 'eZI5q6j8T_fqv7xMROaei9_tmTMk4S7WR5Kr4onPHV8' + '?fpt=ed25519-sha-256&cost=131072') ASSET_DEFINITION = { 'data': { @@ -71,25 +77,25 @@ def cond_uri(): @pytest.fixture def user_Ed25519(user_pub): - from cryptoconditions import Ed25519Fulfillment - return Ed25519Fulfillment(public_key=user_pub) + from cryptoconditions import Ed25519Sha256 + return Ed25519Sha256(public_key=b58decode(user_pub)) @pytest.fixture def user_user2_threshold(user_pub, user2_pub): - from cryptoconditions import (ThresholdSha256Fulfillment, - Ed25519Fulfillment) + from cryptoconditions import ThresholdSha256, Ed25519Sha256 user_pub_keys = [user_pub, user2_pub] - threshold = ThresholdSha256Fulfillment(threshold=len(user_pub_keys)) + threshold = ThresholdSha256(threshold=len(user_pub_keys)) for user_pub in user_pub_keys: - threshold.add_subfulfillment(Ed25519Fulfillment(public_key=user_pub)) + threshold.add_subfulfillment( + Ed25519Sha256(public_key=b58decode(user_pub))) return threshold @pytest.fixture def user2_Ed25519(user2_pub): - from cryptoconditions import Ed25519Fulfillment - return Ed25519Fulfillment(public_key=user2_pub) + from cryptoconditions import Ed25519Sha256 + return Ed25519Sha256(public_key=b58decode(user2_pub)) @pytest.fixture diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 18750ad4..ffe42d1e 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -2,7 +2,9 @@ These are tests of the API of the Transaction class and associated classes. Tests for transaction validation are separate. """ +from copy import deepcopy +from base58 import b58encode, b58decode from pytest import raises @@ -80,7 +82,10 @@ def test_output_serialization(user_Ed25519, user_pub): expected = { 'condition': { 'uri': user_Ed25519.condition_uri, - 'details': user_Ed25519.to_dict(), + 'details': { + 'type': 'ed25519-sha-256', + 'public_key': b58encode(user_Ed25519.public_key), + }, }, 'public_keys': [user_pub], 'amount': '1', @@ -98,7 +103,10 @@ def test_output_deserialization(user_Ed25519, user_pub): cond = { 'condition': { 'uri': user_Ed25519.condition_uri, - 'details': user_Ed25519.to_dict() + 'details': { + 'type': 'ed25519-sha-256', + 'public_key': b58encode(user_Ed25519.public_key), + }, }, 'public_keys': [user_pub], 'amount': '1', @@ -110,10 +118,10 @@ def test_output_deserialization(user_Ed25519, user_pub): def test_output_hashlock_serialization(): from bigchaindb.common.transaction import Output - from cryptoconditions import PreimageSha256Fulfillment + from cryptoconditions import PreimageSha256 secret = b'wow much secret' - hashlock = PreimageSha256Fulfillment(preimage=secret).condition_uri + hashlock = PreimageSha256(preimage=secret).condition_uri expected = { 'condition': { @@ -129,10 +137,10 @@ def test_output_hashlock_serialization(): def test_output_hashlock_deserialization(): from bigchaindb.common.transaction import Output - from cryptoconditions import PreimageSha256Fulfillment + from cryptoconditions import PreimageSha256 secret = b'wow much secret' - hashlock = PreimageSha256Fulfillment(preimage=secret).condition_uri + hashlock = PreimageSha256(preimage=secret).condition_uri expected = Output(hashlock, amount=1) cond = { @@ -161,15 +169,15 @@ def test_invalid_output_initialization(cond_uri, user_pub): def test_generate_output_split_half_recursive(user_pub, user2_pub, user3_pub): from bigchaindb.common.transaction import Output - from cryptoconditions import Ed25519Fulfillment, ThresholdSha256Fulfillment + from cryptoconditions import Ed25519Sha256, ThresholdSha256 - expected_simple1 = Ed25519Fulfillment(public_key=user_pub) - expected_simple2 = Ed25519Fulfillment(public_key=user2_pub) - expected_simple3 = Ed25519Fulfillment(public_key=user3_pub) + expected_simple1 = Ed25519Sha256(public_key=b58decode(user_pub)) + expected_simple2 = Ed25519Sha256(public_key=b58decode(user2_pub)) + expected_simple3 = Ed25519Sha256(public_key=b58decode(user3_pub)) - expected = ThresholdSha256Fulfillment(threshold=2) + expected = ThresholdSha256(threshold=2) expected.add_subfulfillment(expected_simple1) - expected_threshold = ThresholdSha256Fulfillment(threshold=2) + expected_threshold = ThresholdSha256(threshold=2) expected_threshold.add_subfulfillment(expected_simple2) expected_threshold.add_subfulfillment(expected_simple3) expected.add_subfulfillment(expected_threshold) @@ -181,14 +189,14 @@ def test_generate_output_split_half_recursive(user_pub, user2_pub, user3_pub): def test_generate_outputs_split_half_single_owner(user_pub, user2_pub, user3_pub): from bigchaindb.common.transaction import Output - from cryptoconditions import Ed25519Fulfillment, ThresholdSha256Fulfillment + from cryptoconditions import Ed25519Sha256, ThresholdSha256 - expected_simple1 = Ed25519Fulfillment(public_key=user_pub) - expected_simple2 = Ed25519Fulfillment(public_key=user2_pub) - expected_simple3 = Ed25519Fulfillment(public_key=user3_pub) + expected_simple1 = Ed25519Sha256(public_key=b58decode(user_pub)) + expected_simple2 = Ed25519Sha256(public_key=b58decode(user2_pub)) + expected_simple3 = Ed25519Sha256(public_key=b58decode(user3_pub)) - expected = ThresholdSha256Fulfillment(threshold=2) - expected_threshold = ThresholdSha256Fulfillment(threshold=2) + expected = ThresholdSha256(threshold=2) + expected_threshold = ThresholdSha256(threshold=2) expected_threshold.add_subfulfillment(expected_simple2) expected_threshold.add_subfulfillment(expected_simple3) expected.add_subfulfillment(expected_threshold) @@ -200,13 +208,13 @@ def test_generate_outputs_split_half_single_owner(user_pub, def test_generate_outputs_flat_ownage(user_pub, user2_pub, user3_pub): from bigchaindb.common.transaction import Output - from cryptoconditions import Ed25519Fulfillment, ThresholdSha256Fulfillment + from cryptoconditions import Ed25519Sha256, ThresholdSha256 - expected_simple1 = Ed25519Fulfillment(public_key=user_pub) - expected_simple2 = Ed25519Fulfillment(public_key=user2_pub) - expected_simple3 = Ed25519Fulfillment(public_key=user3_pub) + expected_simple1 = Ed25519Sha256(public_key=b58decode(user_pub)) + expected_simple2 = Ed25519Sha256(public_key=b58decode(user2_pub)) + expected_simple3 = Ed25519Sha256(public_key=b58decode(user3_pub)) - expected = ThresholdSha256Fulfillment(threshold=3) + expected = ThresholdSha256(threshold=3) expected.add_subfulfillment(expected_simple1) expected.add_subfulfillment(expected_simple2) expected.add_subfulfillment(expected_simple3) @@ -217,9 +225,9 @@ def test_generate_outputs_flat_ownage(user_pub, user2_pub, user3_pub): def test_generate_output_single_owner(user_pub): from bigchaindb.common.transaction import Output - from cryptoconditions import Ed25519Fulfillment + from cryptoconditions import Ed25519Sha256 - expected = Ed25519Fulfillment(public_key=user_pub) + expected = Ed25519Sha256(public_key=b58decode(user_pub)) cond = Output.generate([user_pub], 1) assert cond.fulfillment.to_dict() == expected.to_dict() @@ -227,9 +235,9 @@ def test_generate_output_single_owner(user_pub): def test_generate_output_single_owner_with_output(user_pub): from bigchaindb.common.transaction import Output - from cryptoconditions import Ed25519Fulfillment + from cryptoconditions import Ed25519Sha256 - expected = Ed25519Fulfillment(public_key=user_pub) + expected = Ed25519Sha256(public_key=b58decode(user_pub)) cond = Output.generate([expected], 1) assert cond.fulfillment.to_dict() == expected.to_dict() @@ -363,8 +371,8 @@ def test_transaction_link_serialization(): tx_id = 'a transaction id' expected = { - 'txid': tx_id, - 'output': 0, + 'transaction_id': tx_id, + 'output_index': 0, } tx_link = TransactionLink(tx_id, 0) @@ -386,8 +394,8 @@ def test_transaction_link_deserialization(): tx_id = 'a transaction id' expected = TransactionLink(tx_id, 0) tx_link = { - 'txid': tx_id, - 'output': 0, + 'transaction_id': tx_id, + 'output_index': 0, } tx_link = TransactionLink.from_dict(tx_link) @@ -489,15 +497,13 @@ def test_sign_with_invalid_parameters(utx, user_priv): def test_validate_tx_simple_create_signature(user_input, user_output, user_priv, asset_definition): - from copy import deepcopy - from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction from .utils import validate_transaction_model tx = Transaction(Transaction.CREATE, asset_definition, [user_input], [user_output]) expected = deepcopy(user_output) message = str(tx).encode() - expected.fulfillment.sign(message, PrivateKey(user_priv)) + expected.fulfillment.sign(message, b58decode(user_priv)) tx.sign([user_priv]) assert tx.inputs[0].to_dict()['fulfillment'] == \ @@ -527,7 +533,7 @@ def test_sign_threshold_with_invalid_params(utx, user_user2_threshold_input, 'somemessage', {user3_pub: user3_priv}) with raises(KeypairMismatchException): - user_user2_threshold_input.owners_before = ['somewrongvalue'] + user_user2_threshold_input.owners_before = [58 * 'a'] utx._sign_threshold_signature_fulfillment(user_user2_threshold_input, 'somemessage', None) @@ -551,9 +557,6 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_input, user_priv, user2_priv, asset_definition): - from copy import deepcopy - - from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction from .utils import validate_transaction_model @@ -562,10 +565,10 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_input, [user_user2_threshold_output]) message = str(tx).encode() expected = deepcopy(user_user2_threshold_output) - expected.fulfillment.subconditions[0]['body'].sign(message, - PrivateKey(user_priv)) - expected.fulfillment.subconditions[1]['body'].sign(message, - PrivateKey(user2_priv)) + expected.fulfillment.subconditions[0]['body'].sign( + message, b58decode(user_priv)) + expected.fulfillment.subconditions[1]['body'].sign( + message, b58decode(user2_priv)) tx.sign([user_priv, user2_priv]) assert tx.inputs[0].to_dict()['fulfillment'] == \ @@ -577,14 +580,14 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_input, def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, asset_definition): - from copy import deepcopy - from cryptoconditions import Ed25519Fulfillment, ThresholdSha256Fulfillment + from cryptoconditions import Ed25519Sha256, ThresholdSha256 from bigchaindb.common.transaction import Input, Output, Transaction - from bigchaindb.common.crypto import PrivateKey - threshold = ThresholdSha256Fulfillment(threshold=2) - threshold.add_subfulfillment(Ed25519Fulfillment(public_key=user_pub)) - threshold.add_subfulfillment(Ed25519Fulfillment(public_key=user_pub)) + threshold = ThresholdSha256(threshold=2) + threshold.add_subfulfillment( + Ed25519Sha256(public_key=b58decode(user_pub))) + threshold.add_subfulfillment( + Ed25519Sha256(public_key=b58decode(user_pub))) threshold_input = Input(threshold, [user_pub, user_pub]) threshold_output = Output(threshold, [user_pub, user_pub]) @@ -592,10 +595,10 @@ def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, tx = Transaction(Transaction.CREATE, asset_definition, [threshold_input], [threshold_output]) expected = deepcopy(threshold_input) - expected.fulfillment.subconditions[0]['body'].sign(str(tx).encode(), - PrivateKey(user_priv)) - expected.fulfillment.subconditions[1]['body'].sign(str(tx).encode(), - PrivateKey(user_priv)) + expected.fulfillment.subconditions[0]['body'].sign( + str(tx).encode(), b58decode(user_priv)) + expected.fulfillment.subconditions[1]['body'].sign( + str(tx).encode(), b58decode(user_priv)) tx.sign([user_priv, user_priv]) @@ -616,10 +619,9 @@ def test_multiple_input_validation_of_transfer_tx(user_input, user_output, user2_priv, user3_pub, user3_priv, asset_definition): - from copy import deepcopy from bigchaindb.common.transaction import (Transaction, TransactionLink, Input, Output) - from cryptoconditions import Ed25519Fulfillment + from cryptoconditions import Ed25519Sha256 from .utils import validate_transaction_model tx = Transaction(Transaction.CREATE, asset_definition, [user_input], @@ -629,8 +631,10 @@ def test_multiple_input_validation_of_transfer_tx(user_input, user_output, inputs = [Input(cond.fulfillment, cond.public_keys, TransactionLink(tx.id, index)) for index, cond in enumerate(tx.outputs)] - outputs = [Output(Ed25519Fulfillment(public_key=user3_pub), [user3_pub]), - Output(Ed25519Fulfillment(public_key=user3_pub), [user3_pub])] + outputs = [Output(Ed25519Sha256(public_key=b58decode(user3_pub)), + [user3_pub]), + Output(Ed25519Sha256(public_key=b58decode(user3_pub)), + [user3_pub])] transfer_tx = Transaction('TRANSFER', {'id': tx.id}, inputs, outputs) transfer_tx = transfer_tx.sign([user_priv]) @@ -640,11 +644,11 @@ def test_multiple_input_validation_of_transfer_tx(user_input, user_output, def test_validate_inputs_of_transfer_tx_with_invalid_params( - transfer_tx, cond_uri, utx, user2_pub, user_priv): + transfer_tx, cond_uri, utx, user2_pub, user_priv, ffill_uri): from bigchaindb.common.transaction import Output - from cryptoconditions import Ed25519Fulfillment + from cryptoconditions import Ed25519Sha256 - invalid_out = Output(Ed25519Fulfillment.from_uri('cf:0:'), ['invalid']) + invalid_out = Output(Ed25519Sha256.from_uri(ffill_uri), ['invalid']) assert transfer_tx.inputs_valid([invalid_out]) is False invalid_out = utx.outputs[0] invalid_out.public_key = 'invalid' @@ -826,8 +830,6 @@ def test_outputs_to_inputs(tx): def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, user2_output, user_priv): - from copy import deepcopy - from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import serialize from .utils import validate_transaction_model @@ -845,8 +847,8 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, ], 'fulfillment': None, 'fulfills': { - 'txid': tx.id, - 'output': 0 + 'transaction_id': tx.id, + 'output_index': 0 } } ], @@ -861,8 +863,8 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, expected_input = deepcopy(inputs[0]) expected['id'] = transfer_tx['id'] - expected_input.fulfillment.sign(serialize(expected).encode(), - PrivateKey(user_priv)) + expected_input.fulfillment.sign( + serialize(expected).encode(), b58decode(user_priv)) expected_ffill = expected_input.fulfillment.serialize_uri() transfer_ffill = transfer_tx['inputs'][0]['fulfillment'] @@ -894,8 +896,8 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, ], 'fulfillment': None, 'fulfills': { - 'txid': tx.id, - 'output': 0 + 'transaction_id': tx.id, + 'output_index': 0 } }, { 'owners_before': [ @@ -903,8 +905,8 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, ], 'fulfillment': None, 'fulfills': { - 'txid': tx.id, - 'output': 1 + 'transaction_id': tx.id, + 'output_index': 1 } } ], diff --git a/tests/conftest.py b/tests/conftest.py index d60b4511..3a1ace15 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,9 +26,10 @@ USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' def pytest_addoption(parser): - from bigchaindb.backend import connection + from bigchaindb.backend.connection import BACKENDS - backends = ', '.join(connection.BACKENDS.keys()) + BACKENDS['mongodb-ssl'] = 'bigchaindb.backend.mongodb.connection.MongoDBConnection' + backends = ', '.join(BACKENDS.keys()) parser.addoption( '--database-backend', action='store', @@ -41,9 +42,12 @@ def pytest_ignore_collect(path, config): from bigchaindb.backend.connection import BACKENDS path = str(path) + BACKENDS['mongodb-ssl'] = 'bigchaindb.backend.mongodb.connection.MongoDBConnection' + supported_backends = BACKENDS.keys() + if os.path.isdir(path): dirname = os.path.split(path)[1] - if dirname in BACKENDS.keys() and dirname != config.getoption('--database-backend'): + if dirname in supported_backends and dirname != config.getoption('--database-backend'): print('Ignoring unrequested backend test dir: ', path) return True @@ -110,7 +114,7 @@ def _restore_dbs(request): @pytest.fixture(scope='session') -def _configure_bigchaindb(request): +def _configure_bigchaindb(request, certs_dir): import bigchaindb from bigchaindb import config_utils test_db_name = TEST_DB_NAME @@ -120,6 +124,22 @@ def _configure_bigchaindb(request): test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) backend = request.config.getoption('--database-backend') + + if backend == 'mongodb-ssl': + bigchaindb._database_map[backend] = { + # we use mongodb as the backend for mongodb-ssl + 'backend': 'mongodb', + 'connection_timeout': 5000, + 'max_tries': 3, + 'ssl': True, + 'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT', certs_dir + '/ca.crt'), + 'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE', certs_dir + '/crl.pem'), + 'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE', certs_dir + '/test_bdb_ssl.crt'), + 'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE', certs_dir + '/test_bdb_ssl.key'), + 'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE', None) + } + bigchaindb._database_map[backend].update(bigchaindb._base_database_mongodb) + config = { 'database': bigchaindb._database_map[backend], 'keypair': { @@ -454,3 +474,9 @@ def mocked_setup_pub_logger(mocker): def mocked_setup_sub_logger(mocker): return mocker.patch( 'bigchaindb.log.setup.setup_sub_logger', autospec=True, spec_set=True) + + +@pytest.fixture(scope='session') +def certs_dir(): + cwd = os.environ.get('TRAVIS_BUILD_DIR', os.getcwd()) + return cwd + '/tests/backend/mongodb-ssl/certs' diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 05b07bf6..34374837 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1,7 +1,8 @@ from time import sleep +from unittest.mock import patch import pytest -from unittest.mock import patch +from base58 import b58decode pytestmark = pytest.mark.bdb @@ -577,14 +578,14 @@ class TestBigchainApi(object): @pytest.mark.usefixtures('inputs') def test_non_create_input_not_found(self, b, user_pk): - from cryptoconditions import Ed25519Fulfillment + from cryptoconditions import Ed25519Sha256 from bigchaindb.common.exceptions import InputDoesNotExist from bigchaindb.common.transaction import Input, TransactionLink from bigchaindb.models import Transaction from bigchaindb import Bigchain # Create an input for a non existing transaction - input = Input(Ed25519Fulfillment(public_key=user_pk), + input = Input(Ed25519Sha256(public_key=b58decode(user_pk)), [user_pk], TransactionLink('somethingsomething', 0)) tx = Transaction.transfer([input], [([user_pk], 1)], @@ -1194,7 +1195,7 @@ def test_get_owned_ids_calls_get_outputs_filtered(): with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: b = Bigchain() res = b.get_owned_ids('abc') - gof.assert_called_once_with('abc', include_spent=False) + gof.assert_called_once_with('abc', spent=False) assert res == gof() @@ -1206,21 +1207,36 @@ def test_get_outputs_filtered_only_unspent(): TransactionLink('b', 2)] with patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') as filter_spent: filter_spent.return_value = [TransactionLink('b', 2)] - out = Bigchain().get_outputs_filtered('abc', include_spent=False) + out = Bigchain().get_outputs_filtered('abc', spent=False) get_outputs.assert_called_once_with('abc') assert out == [TransactionLink('b', 2)] -def test_get_outputs_filtered(): +def test_get_outputs_filtered_only_spent(): from bigchaindb.common.transaction import TransactionLink from bigchaindb.core import Bigchain with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - with patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') as filter_spent: - out = Bigchain().get_outputs_filtered('abc') + with patch('bigchaindb.fastquery.FastQuery.filter_unspent_outputs') as filter_spent: + filter_spent.return_value = [TransactionLink('b', 2)] + out = Bigchain().get_outputs_filtered('abc', spent=True) + get_outputs.assert_called_once_with('abc') + assert out == [TransactionLink('b', 2)] + + +@patch('bigchaindb.fastquery.FastQuery.filter_unspent_outputs') +@patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') +def test_get_outputs_filtered(filter_spent, filter_unspent): + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.core import Bigchain + with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: + get_outputs.return_value = [TransactionLink('a', 1), + TransactionLink('b', 2)] + out = Bigchain().get_outputs_filtered('abc') get_outputs.assert_called_once_with('abc') filter_spent.assert_not_called() + filter_unspent.assert_not_called() assert out == get_outputs.return_value diff --git a/tests/integration/test_federation.py b/tests/integration/test_federation.py index 00c59685..598412ff 100644 --- a/tests/integration/test_federation.py +++ b/tests/integration/test_federation.py @@ -133,27 +133,6 @@ def test_elect_invalid(federation_3): assert bx[i].get_transaction(tx.id, True)[1] is None -@pytest.mark.bdb -@pytest.mark.genesis -def test_elect_disagree_prev_block(federation_3): - [bx, (s0, s1, s2)] = federation_3 - tx = input_single_create(bx[0]) - process_tx(s0) - process_tx(s1) - process_tx(s2) - process_vote(s0, True) - for i in range(3): - assert bx[i].get_transaction(tx.id, True)[1] == 'undecided' - s1.vote.last_voted_id = '5' * 64 - process_vote(s1, True) - for i in range(3): - assert bx[i].get_transaction(tx.id, True)[1] == 'undecided' - s2.vote.last_voted_id = '6' * 64 - process_vote(s2, True) - for i in range(3): - assert bx[i].get_transaction(tx.id, True)[1] is None - - @pytest.mark.bdb @pytest.mark.genesis def test_elect_sybill(federation_3): diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index f68c6f6e..21bbc0bf 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -180,7 +180,7 @@ def test_vote_accumulates_transactions(b): validation = vote_obj.validate_tx(tx.to_dict(), 123, 1) assert validation == (True, 123, 1) - tx.inputs[0].fulfillment.signature = None + tx.inputs[0].fulfillment.signature = 64*b'z' validation = vote_obj.validate_tx(tx.to_dict(), 456, 10) assert validation == (False, 456, 10) @@ -199,7 +199,7 @@ def test_valid_block_voting_sequential(b, genesis_block, monkeypatch): for tx, block_id, num_tx in vote_obj.ungroup(block['id'], txs): last_vote = vote_obj.vote(*vote_obj.validate_tx(tx, block_id, num_tx)) - vote_obj.write_vote(last_vote) + vote_obj.write_vote(*last_vote) vote_rs = query.get_votes_by_block_id_and_voter(b.connection, block_id, b.me) vote_doc = vote_rs.next() diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index a2d1f13e..72f09039 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -15,6 +15,8 @@ def clean_config(monkeypatch, request): import bigchaindb original_config = copy.deepcopy(ORIGINAL_CONFIG) backend = request.config.getoption('--database-backend') + if backend == 'mongodb-ssl': + backend = 'mongodb' original_config['database'] = bigchaindb._database_map[backend] monkeypatch.setattr('bigchaindb.config', original_config) @@ -138,13 +140,14 @@ def test_env_config(monkeypatch): assert result == expected -def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): +def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request, certs_dir): # constants DATABASE_HOST = 'test-host' DATABASE_NAME = 'test-dbname' DATABASE_PORT = 4242 DATABASE_BACKEND = request.config.getoption('--database-backend') SERVER_BIND = '1.2.3.4:56' + WSSERVER_SCHEME = 'ws' WSSERVER_HOST = '1.2.3.4' WSSERVER_PORT = 57 KEYRING = 'pubkey_0:pubkey_1:pubkey_2' @@ -159,15 +162,34 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'level_console': 'debug', }, } + monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) - monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': DATABASE_NAME, - 'BIGCHAINDB_DATABASE_PORT': str(DATABASE_PORT), - 'BIGCHAINDB_DATABASE_BACKEND': DATABASE_BACKEND, - 'BIGCHAINDB_SERVER_BIND': SERVER_BIND, - 'BIGCHAINDB_WSSERVER_HOST': WSSERVER_HOST, - 'BIGCHAINDB_WSSERVER_PORT': WSSERVER_PORT, - 'BIGCHAINDB_KEYRING': KEYRING, - 'BIGCHAINDB_LOG_FILE': LOG_FILE}) + + if DATABASE_BACKEND == 'mongodb-ssl': + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': DATABASE_NAME, + 'BIGCHAINDB_DATABASE_PORT': str(DATABASE_PORT), + 'BIGCHAINDB_DATABASE_BACKEND': 'mongodb', + 'BIGCHAINDB_SERVER_BIND': SERVER_BIND, + 'BIGCHAINDB_WSSERVER_SCHEME': WSSERVER_SCHEME, + 'BIGCHAINDB_WSSERVER_HOST': WSSERVER_HOST, + 'BIGCHAINDB_WSSERVER_PORT': WSSERVER_PORT, + 'BIGCHAINDB_KEYRING': KEYRING, + 'BIGCHAINDB_LOG_FILE': LOG_FILE, + 'BIGCHAINDB_DATABASE_CA_CERT': certs_dir + '/ca.crt', + 'BIGCHAINDB_DATABASE_CRLFILE': certs_dir + '/crl.pem', + 'BIGCHAINDB_DATABASE_CERTFILE': certs_dir + '/test_bdb_ssl.crt', + 'BIGCHAINDB_DATABASE_KEYFILE': certs_dir + '/test_bdb_ssl.key', + 'BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE': None}) + else: + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': DATABASE_NAME, + 'BIGCHAINDB_DATABASE_PORT': str(DATABASE_PORT), + 'BIGCHAINDB_DATABASE_BACKEND': DATABASE_BACKEND, + 'BIGCHAINDB_SERVER_BIND': SERVER_BIND, + 'BIGCHAINDB_WSSERVER_SCHEME': WSSERVER_SCHEME, + 'BIGCHAINDB_WSSERVER_HOST': WSSERVER_HOST, + 'BIGCHAINDB_WSSERVER_PORT': WSSERVER_PORT, + 'BIGCHAINDB_KEYRING': KEYRING, + 'BIGCHAINDB_LOG_FILE': LOG_FILE}) import bigchaindb from bigchaindb import config_utils @@ -193,7 +215,30 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'replicaset': 'bigchain-rs', 'ssl': False, 'login': None, - 'password': None + 'password': None, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None + } + + database_mongodb_ssl = { + 'backend': 'mongodb', + 'host': DATABASE_HOST, + 'port': DATABASE_PORT, + 'name': DATABASE_NAME, + 'connection_timeout': 5000, + 'max_tries': 3, + 'replicaset': 'bigchain-rs', + 'ssl': True, + 'login': None, + 'password': None, + 'ca_cert': certs_dir + '/ca.crt', + 'crlfile': certs_dir + '/crl.pem', + 'certfile': certs_dir + '/test_bdb_ssl.crt', + 'keyfile': certs_dir + '/test_bdb_ssl.key', + 'keyfile_passphrase': None } database = {} @@ -201,6 +246,8 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): database = database_mongodb elif DATABASE_BACKEND == 'rethinkdb': database = database_rethinkdb + elif DATABASE_BACKEND == 'mongodb-ssl': + database = database_mongodb_ssl assert bigchaindb.config == { 'CONFIGURED': True, @@ -211,6 +258,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'workers': None, }, 'wsserver': { + 'scheme': WSSERVER_SCHEME, 'host': WSSERVER_HOST, 'port': WSSERVER_PORT, }, @@ -233,6 +281,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'fmt_logfile': log_config['formatters']['file']['format'], 'granular_levels': {}, }, + 'graphite': {'host': 'localhost'}, } diff --git a/tests/test_core.py b/tests/test_core.py index b8803e9b..7a369bdd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3,9 +3,13 @@ import pytest @pytest.fixture def config(request, monkeypatch): + backend = request.config.getoption('--database-backend') + if backend == 'mongodb-ssl': + backend = 'mongodb' + config = { 'database': { - 'backend': request.config.getoption('--database-backend'), + 'backend': backend, 'host': 'host', 'port': 28015, 'name': 'bigchain', @@ -19,7 +23,8 @@ def config(request, monkeypatch): }, 'keyring': [], 'CONFIGURED': True, - 'backlog_reassign_delay': 30 + 'backlog_reassign_delay': 30, + 'graphite': {'host': 'localhost'}, } monkeypatch.setattr('bigchaindb.config', config) @@ -122,4 +127,4 @@ def test_get_spent_issue_1271(b, alice, bob, carol): assert b.get_spent(tx_2.id, 0) == tx_5 assert not b.get_spent(tx_5.id, 0) assert b.get_outputs_filtered(alice.public_key) - assert b.get_outputs_filtered(alice.public_key, include_spent=False) + assert b.get_outputs_filtered(alice.public_key, spent=False) diff --git a/tests/test_fastquery.py b/tests/test_fastquery.py index 8fb3378c..bf3facf0 100644 --- a/tests/test_fastquery.py +++ b/tests/test_fastquery.py @@ -84,3 +84,39 @@ def test_filter_spent_outputs(b, user_pk): tx2.to_inputs()[0].fulfills, tx4.to_inputs()[0].fulfills } + + +def test_filter_unspent_outputs(b, user_pk): + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 3) + + # There are 3 inputs + inputs = tx1.to_inputs() + + # Each spent individually + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx3 = Transaction.transfer([inputs[1]], out, tx1.id) + tx4 = Transaction.transfer([inputs[2]], out, tx1.id) + + # The CREATE and first TRANSFER are valid. tx2 produces a new unspent. + for tx in [tx1, tx2]: + block = Block([tx]) + b.write_block(block) + b.write_vote(b.vote(block.id, '', True)) + + # The second TRANSFER is invalid. inputs[1] remains unspent. + block = Block([tx3]) + b.write_block(block) + b.write_vote(b.vote(block.id, '', False)) + + # The third TRANSFER is undecided. It procuces a new unspent. + block = Block([tx4]) + b.write_block(block) + + outputs = b.fastquery.get_outputs_by_public_key(user_pk) + spents = b.fastquery.filter_unspent_outputs(outputs) + + assert set(spents) == { + inputs[0].fulfills, + inputs[2].fulfills + } diff --git a/tests/validation/test_transaction_structure.py b/tests/validation/test_transaction_structure.py index 71ba2e5b..22c09651 100644 --- a/tests/validation/test_transaction_structure.py +++ b/tests/validation/test_transaction_structure.py @@ -5,9 +5,11 @@ structural / schematic issues are caught when reading a transaction """ import pytest +from unittest.mock import MagicMock from bigchaindb.common.exceptions import (AmountError, InvalidHash, - SchemaValidationError) + SchemaValidationError, + ThresholdTooDeep) from bigchaindb.models import Transaction @@ -101,6 +103,15 @@ def test_create_tx_asset_type(create_tx): validate_raises(create_tx) +def test_create_tx_no_asset_data(create_tx): + tx_body = create_tx.to_dict() + del tx_body['asset']['data'] + tx_body_no_signatures = Transaction._remove_signatures(tx_body) + tx_body_serialized = Transaction._to_str(tx_body_no_signatures) + tx_body['id'] = Transaction._to_hash(tx_body_serialized) + validate_raises(tx_body) + + ################################################################################ # Inputs @@ -152,18 +163,44 @@ def test_high_amounts(create_tx): validate(create_tx) +################################################################################ +# Conditions + +def test_handle_threshold_overflow(): + from bigchaindb.common import transaction + + cond = { + 'type': 'ed25519-sha-256', + 'public_key': 'a' * 43, + } + for i in range(1000): + cond = { + 'type': 'threshold-sha-256', + 'threshold': 1, + 'subconditions': [cond], + } + with pytest.raises(ThresholdTooDeep): + transaction._fulfillment_from_details(cond) + + +def test_unsupported_condition_type(): + from bigchaindb.common import transaction + from cryptoconditions.exceptions import UnsupportedTypeError + + with pytest.raises(UnsupportedTypeError): + transaction._fulfillment_from_details({'type': 'a'}) + + with pytest.raises(UnsupportedTypeError): + transaction._fulfillment_to_details(MagicMock(type_name='a')) + + ################################################################################ # Version def test_validate_version(create_tx): - import re - import bigchaindb.version - - short_ver = bigchaindb.version.__short_version__ - assert create_tx.version == re.match(r'^(.*\d)', short_ver).group(1) - + create_tx.version = '1.0' validate(create_tx) - - # At version 1, transaction version will break step with server version. - create_tx.version = '1.0.0' + create_tx.version = '0.10' + validate_raises(create_tx) + create_tx.version = '110' validate_raises(create_tx) diff --git a/tests/web/test_blocks.py b/tests/web/test_blocks.py index 01c17d71..3c3270ad 100644 --- a/tests/web/test_blocks.py +++ b/tests/web/test_blocks.py @@ -41,7 +41,7 @@ def test_get_blocks_by_txid_endpoint(b, client): block_invalid = b.create_block([tx]) b.write_block(block_invalid) - res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=' + tx.id) # test if block is retrieved as undecided assert res.status_code == 200 assert block_invalid.id in res.json @@ -51,7 +51,7 @@ def test_get_blocks_by_txid_endpoint(b, client): vote = b.vote(block_invalid.id, b.get_last_voted_block().id, False) b.write_vote(vote) - res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=' + tx.id) # test if block is retrieved as invalid assert res.status_code == 200 assert block_invalid.id in res.json @@ -61,7 +61,7 @@ def test_get_blocks_by_txid_endpoint(b, client): block_valid = b.create_block([tx, tx2]) b.write_block(block_valid) - res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=' + tx.id) # test if block is retrieved as undecided assert res.status_code == 200 assert block_valid.id in res.json @@ -71,7 +71,7 @@ def test_get_blocks_by_txid_endpoint(b, client): vote = b.vote(block_valid.id, block_invalid.id, True) b.write_vote(vote) - res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=' + tx.id) # test if block is retrieved as valid assert res.status_code == 200 assert block_valid.id in res.json @@ -96,19 +96,19 @@ def test_get_blocks_by_txid_and_status_endpoint(b, client): block_valid = b.create_block([tx, tx2]) b.write_block(block_valid) - res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) + res = client.get('{}?transaction_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) # test if no blocks are retrieved as invalid assert res.status_code == 200 assert len(res.json) == 0 - res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) + res = client.get('{}?transaction_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) # test if both blocks are retrieved as undecided assert res.status_code == 200 assert block_valid.id in res.json assert block_invalid.id in res.json assert len(res.json) == 2 - res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) + res = client.get('{}?transaction_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) # test if no blocks are retrieved as valid assert res.status_code == 200 assert len(res.json) == 0 @@ -121,18 +121,18 @@ def test_get_blocks_by_txid_and_status_endpoint(b, client): vote = b.vote(block_valid.id, block_invalid.id, True) b.write_vote(vote) - res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) + res = client.get('{}?transaction_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) # test if the invalid block is retrieved as invalid assert res.status_code == 200 assert block_invalid.id in res.json assert len(res.json) == 1 - res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) + res = client.get('{}?transaction_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) # test if no blocks are retrieved as undecided assert res.status_code == 200 assert len(res.json) == 0 - res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) + res = client.get('{}?transaction_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) # test if the valid block is retrieved as valid assert res.status_code == 200 assert block_valid.id in res.json @@ -141,11 +141,11 @@ def test_get_blocks_by_txid_and_status_endpoint(b, client): @pytest.mark.bdb def test_get_blocks_by_txid_endpoint_returns_empty_list_not_found(client): - res = client.get(BLOCKS_ENDPOINT + '?tx_id=') + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=') assert res.status_code == 200 assert len(res.json) == 0 - res = client.get(BLOCKS_ENDPOINT + '?tx_id=123') + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123') assert res.status_code == 200 assert len(res.json) == 0 @@ -159,17 +159,17 @@ def test_get_blocks_by_txid_endpoint_returns_400_bad_query_params(client): assert res.status_code == 400 assert res.json == { 'message': { - 'tx_id': 'Missing required parameter in the JSON body or the post body or the query string' + 'transaction_id': 'Missing required parameter in the JSON body or the post body or the query string' } } - res = client.get(BLOCKS_ENDPOINT + '?tx_id=123&foo=123') + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123&foo=123') assert res.status_code == 400 assert res.json == { 'message': 'Unknown arguments: foo' } - res = client.get(BLOCKS_ENDPOINT + '?tx_id=123&status=123') + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123&status=123') assert res.status_code == 400 assert res.json == { 'message': { diff --git a/tests/web/test_info.py b/tests/web/test_info.py index 292b1b74..f3a1fe76 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -6,11 +6,20 @@ from unittest import mock @mock.patch('bigchaindb.config', {'keyring': ['abc'], 'keypair': {'public': 'def'}}) def test_api_root_endpoint(client): res = client.get('/') + docs_url = ['https://docs.bigchaindb.com/projects/server/en/vtsttst', + '/http-client-server-api.html'] assert res.json == { - '_links': { - 'docs': 'https://docs.bigchaindb.com/projects/server/en/vtsttst/', - 'api_v1': 'http://localhost/api/v1/', + 'api': { + 'v1': { + 'docs': ''.join(docs_url), + 'transactions': '/api/v1/transactions/', + 'statuses': '/api/v1/statuses/', + 'assets': '/api/v1/assets/', + 'outputs': '/api/v1/outputs/', + 'streams': 'ws://localhost:9985/api/v1/streams/valid_transactions', + } }, + 'docs': 'https://docs.bigchaindb.com/projects/server/en/vtsttst/', 'version': 'tsttst', 'keyring': ['abc'], 'public_key': 'def', @@ -21,16 +30,15 @@ def test_api_root_endpoint(client): @mock.patch('bigchaindb.version.__short_version__', 'tst') @mock.patch('bigchaindb.version.__version__', 'tsttst') def test_api_v1_endpoint(client): - res = client.get('/api/v1') docs_url = ['https://docs.bigchaindb.com/projects/server/en/vtsttst', - '/http-client-server-api.html', - ] - assert res.json == { - '_links': { - 'docs': ''.join(docs_url), - 'self': 'http://localhost/api/v1/', - 'statuses': 'http://localhost/api/v1/statuses/', - 'transactions': 'http://localhost/api/v1/transactions/', - 'streams_v1': 'ws://localhost:9985/api/v1/streams/valid_tx', - } + '/http-client-server-api.html'] + api_v1_info = { + 'docs': ''.join(docs_url), + 'transactions': '/transactions/', + 'statuses': '/statuses/', + 'assets': '/assets/', + 'outputs': '/outputs/', + 'streams': 'ws://localhost:9985/api/v1/streams/valid_transactions', } + res = client.get('/api/v1') + assert res.json == api_v1_info diff --git a/tests/web/test_outputs.py b/tests/web/test_outputs.py index b5a02f76..8ef90b73 100644 --- a/tests/web/test_outputs.py +++ b/tests/web/test_outputs.py @@ -8,27 +8,45 @@ OUTPUTS_ENDPOINT = '/api/v1/outputs/' def test_get_outputs_endpoint(client, user_pk): m = MagicMock() - m.to_uri.side_effect = lambda s: 'a%sb' % s + m.txid = 'a' + m.output = 0 with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: gof.return_value = [m, m] res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk)) - assert res.json == ['a..b', 'a..b'] + assert res.json == [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 0} + ] assert res.status_code == 200 - gof.assert_called_once_with(user_pk, True) + gof.assert_called_once_with(user_pk, None) def test_get_outputs_endpoint_unspent(client, user_pk): m = MagicMock() - m.to_uri.side_effect = lambda s: 'a%sb' % s + m.txid = 'a' + m.output = 0 with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: gof.return_value = [m] - params = '?unspent=true&public_key={}'.format(user_pk) + params = '?spent=False&public_key={}'.format(user_pk) res = client.get(OUTPUTS_ENDPOINT + params) - assert res.json == ['a..b'] + assert res.json == [{'transaction_id': 'a', 'output_index': 0}] assert res.status_code == 200 gof.assert_called_once_with(user_pk, False) +def test_get_outputs_endpoint_spent(client, user_pk): + m = MagicMock() + m.txid = 'a' + m.output = 0 + with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: + gof.return_value = [m] + params = '?spent=true&public_key={}'.format(user_pk) + res = client.get(OUTPUTS_ENDPOINT + params) + assert res.json == [{'transaction_id': 'a', 'output_index': 0}] + assert res.status_code == 200 + gof.assert_called_once_with(user_pk, True) + + def test_get_outputs_endpoint_without_public_key(client): res = client.get(OUTPUTS_ENDPOINT) assert res.status_code == 400 @@ -41,9 +59,9 @@ def test_get_outputs_endpoint_with_invalid_public_key(client): assert res.status_code == 400 -def test_get_outputs_endpoint_with_invalid_unspent(client, user_pk): - expected = {'message': {'unspent': 'Boolean value must be "true" or "false" (lowercase)'}} - params = '?unspent=tru&public_key={}'.format(user_pk) +def test_get_outputs_endpoint_with_invalid_spent(client, user_pk): + expected = {'message': {'spent': 'Boolean value must be "true" or "false" (lowercase)'}} + params = '?spent=tru&public_key={}'.format(user_pk) res = client.get(OUTPUTS_ENDPOINT + params) assert expected == res.json assert res.status_code == 400 diff --git a/tests/web/test_statuses.py b/tests/web/test_statuses.py index 716cc0d2..7c2a7c14 100644 --- a/tests/web/test_statuses.py +++ b/tests/web/test_statuses.py @@ -10,15 +10,14 @@ STATUSES_ENDPOINT = '/api/v1/statuses' def test_get_transaction_status_endpoint(b, client, user_pk): input_tx = b.get_owned_ids(user_pk).pop() tx, status = b.get_transaction(input_tx.txid, include_status=True) - res = client.get(STATUSES_ENDPOINT + '?tx_id=' + input_tx.txid) + res = client.get(STATUSES_ENDPOINT + '?transaction_id=' + input_tx.txid) assert status == res.json['status'] - assert res.json['_links']['tx'] == '/transactions/{}'.format(input_tx.txid) assert res.status_code == 200 @pytest.mark.bdb def test_get_transaction_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + '?tx_id=123') + res = client.get(STATUSES_ENDPOINT + '?transaction_id=123') assert res.status_code == 404 @@ -34,7 +33,6 @@ def test_get_block_status_endpoint_undecided(b, client): res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) assert status == res.json['status'] - assert '_links' not in res.json assert res.status_code == 200 @@ -55,7 +53,6 @@ def test_get_block_status_endpoint_valid(b, client): res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) assert status == res.json['status'] - assert '_links' not in res.json assert res.status_code == 200 @@ -76,7 +73,6 @@ def test_get_block_status_endpoint_invalid(b, client): res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) assert status == res.json['status'] - assert '_links' not in res.json assert res.status_code == 200 @@ -94,5 +90,5 @@ def test_get_status_endpoint_returns_400_bad_query_params(client): res = client.get(STATUSES_ENDPOINT + '?ts_id=123') assert res.status_code == 400 - res = client.get(STATUSES_ENDPOINT + '?tx_id=123&block_id=123') + res = client.get(STATUSES_ENDPOINT + '?transaction_id=123&block_id=123') assert res.status_code == 400 diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 4c6e76c1..03eaaa3e 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -40,6 +40,9 @@ def test_post_create_transaction_endpoint(b, client): assert res.status_code == 202 + assert '../statuses?transaction_id={}'.format(tx.id) in \ + res.headers['Location'] + assert res.json['inputs'][0]['owners_before'][0] == user_pub assert res.json['outputs'][0]['public_keys'][0] == user_pub @@ -86,7 +89,7 @@ def test_post_create_transaction_with_invalid_signature(mock_logger, tx = Transaction.create([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv]).to_dict() - tx['inputs'][0]['fulfillment'] = 'cf:0:0' + tx['inputs'][0]['fulfillment'] = 64 * '0' res = client.post(TX_ENDPOINT, data=json.dumps(tx)) expected_status_code = 400 @@ -270,3 +273,26 @@ def test_transactions_get_list_bad(client): # Test asset ID required url = TX_ENDPOINT + '?operation=CREATE' assert client.get(url).status_code == 400 + + +def test_return_only_valid_transaction(client): + from bigchaindb import Bigchain + + def get_transaction_patched(status): + def inner(self, tx_id, include_status): + return {}, status + return inner + + # NOTE: `get_transaction` only returns a transaction if it's included in an + # UNDECIDED or VALID block, as well as transactions from the backlog. + # As the endpoint uses `get_transaction`, we don't have to test + # against invalid transactions here. + with patch('bigchaindb.core.Bigchain.get_transaction', + get_transaction_patched(Bigchain.TX_UNDECIDED)): + url = '{}{}'.format(TX_ENDPOINT, '123') + assert client.get(url).status_code == 404 + + with patch('bigchaindb.core.Bigchain.get_transaction', + get_transaction_patched(Bigchain.TX_IN_BACKLOG)): + url = '{}{}'.format(TX_ENDPOINT, '123') + assert client.get(url).status_code == 404 diff --git a/tests/web/test_websocket_server.py b/tests/web/test_websocket_server.py index f25e183f..d071f9e7 100644 --- a/tests/web/test_websocket_server.py +++ b/tests/web/test_websocket_server.py @@ -132,7 +132,7 @@ def test_websocket_block_event(b, _block, test_client, loop): for tx in block['block']['transactions']: result = yield from ws.receive() json_result = json.loads(result.data) - assert json_result['tx_id'] == tx['id'] + assert json_result['transaction_id'] == tx['id'] # Since the transactions are all CREATEs, asset id == transaction id assert json_result['asset_id'] == tx['id'] assert json_result['block_id'] == block['id'] @@ -184,4 +184,4 @@ def test_integration_from_webapi_to_websocket(monkeypatch, client, loop): result = loop.run_until_complete(ws.receive()) json_result = json.loads(result.data) - assert json_result['tx_id'] == tx.id + assert json_result['transaction_id'] == tx.id diff --git a/tox.ini b/tox.ini index bdaea034..d92c84c5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = true -envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver +envlist = py{35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver [base] basepython = python3.6 @@ -13,6 +13,7 @@ setenv = rethinkdb: BIGCHAINDB_DATABASE_BACKEND=rethinkdb mongodb: BIGCHAINDB_DATABASE_BACKEND=mongodb deps = {[base]deps} +install_command = pip install {opts} {packages} extras = test commands = pytest -v -n auto --cov=bigchaindb --basetemp={envtmpdir}