diff --git a/.ci/entrypoint.sh b/.ci/entrypoint.sh new file mode 100755 index 0000000..6a96dd7 --- /dev/null +++ b/.ci/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -e -x + +if [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then + sleep 3600 +else + bigchaindb -l DEBUG start +fi diff --git a/.ci/travis-after-success.sh b/.ci/travis-after-success.sh new file mode 100755 index 0000000..af77412 --- /dev/null +++ b/.ci/travis-after-success.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -e -x + +if [[ -z ${TOXENV} ]] && [[ ${PLANETMINT_CI_ABCI} != 'enable' ]] && [[ ${PLANETMINT_ACCEPTANCE_TEST} != 'enable' ]]; then + codecov -v -f htmlcov/coverage.xml +fi diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh new file mode 100755 index 0000000..384dcc2 --- /dev/null +++ b/.ci/travis-before-install.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +if [[ -z ${TOXENV} ]]; then + sudo apt-get update + sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + + sudo rm /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + chmod +x docker-compose + sudo mv docker-compose /usr/local/bin +fi diff --git a/.ci/travis-before-script.sh b/.ci/travis-before-script.sh new file mode 100755 index 0000000..0bf784b --- /dev/null +++ b/.ci/travis-before-script.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -e -x + +if [[ -z ${TOXENV} ]]; then + + if [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then + docker-compose up -d bigchaindb + else + docker-compose up -d bdb + fi + +fi diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh new file mode 100755 index 0000000..1462505 --- /dev/null +++ b/.ci/travis-install.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -e -x + +pip install --upgrade pip + +if [[ -n ${TOXENV} ]]; then + pip install --upgrade tox +elif [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then + docker-compose build --no-cache --build-arg abci_status=enable bigchaindb +elif [[ $PLANETMINT_INTEGRATION_TEST == 'enable' ]]; then + docker-compose build bigchaindb python-driver +else + docker-compose build --no-cache bigchaindb + pip install --upgrade codecov +fi diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh new file mode 100755 index 0000000..a6af3b9 --- /dev/null +++ b/.ci/travis_script.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -e -x + +if [[ -n ${TOXENV} ]]; then + tox -e ${TOXENV} +elif [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then + docker-compose exec bigchaindb pytest -v -m abci +elif [[ ${PLANETMINT_ACCEPTANCE_TEST} == 'enable' ]]; then + ./run-acceptance-test.sh +else + docker-compose exec bigchaindb pytest -v --cov=bigchaindb --cov-report xml:htmlcov/coverage.xml +fi diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..441eb44 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.cache/ +.coverage +.eggs/ +.git/ +.gitignore +.ropeproject/ +.travis.yml +Planetmint.egg-info/ +dist/ diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..e78d110 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,14 @@ + + +# How to Contribute to the Planetmint Project + +There are many ways you can contribute to the Planetmint project, some very easy and others more involved. + +All of that is documented elsewhere: go to the "[Contributing to Planetmint" docs on ReadTheDocs](https://docs.bigchaindb.com/projects/contributing/en/latest/index.html). + +Note: GitHub automatically links to this file (`.github/CONTRIBUTING.md`) when a contributor creates a new issue or pull request, so you shouldn't delete it. Just use it to point people to full and proper help elsewhere. diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..cf0aff9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,20 @@ +# Do you want to: + +- make a bug report? Then read below about what should go in a bug report. +- make a feature request or proposal? Then read [the page about how to make a feature request or proposal](https://docs.bigchaindb.com/projects/contributing/en/latest/ways-can-contribute/make-a-feature-request-or-proposal.html). +- ask a question about Planetmint? Then [go to Gitter](https://gitter.im/bigchaindb/bigchaindb) (our chat room) and ask it there. +- share your neat idea or realization? Then [go to Gitter](https://gitter.im/bigchaindb/bigchaindb) (our chat room) and share it there. + +# What Should Go in a Bug Report + +- What computer are you on (hardware)? +- What operating system are you using, including version. e.g. Ubuntu 14.04? Fedora 23? +- What version of Planetmint software were you using? Is that the latest version? +- What, exactly, did you do to get to the point where you got stuck? Describe all the steps so we can get there too. Show screenshots or copy-and-paste text to GitHub. +- Show what actually happened. +- Say what you tried to do to resolve the problem. +- Provide details to convince us that it matters to you. Is it for a school project, a job, a contract with a deadline, a child who needs it for Christmas? + +We will do our best but please understand that we don't have time to help everyone, especially people who don't care to help us help them. "It doesn't work." is not going to get any reaction from us. We need _details_. + +Tip: Use Github code block formatting to make code render pretty in GitHub. To do that, put three backticks followed by a string to set the type of code (e.g. `Python`), then the code, and then end with three backticks. There's more information about [inserting code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks/) in the GitHub help pages. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..8c269fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Logs or terminal output** +If applicable, add add textual content to help explain your problem. + +**Desktop (please complete the following information):** + - Distribution: [e.g. Ubuntu 18.04] + - Bigchaindb version: + - Tendermint version: + - Mongodb version: +- Python full version: [e.g. Python 3.6.6] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..9305a8a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +Make sure the title of this pull request has the form: + +**Problem: A short statement of the problem.** + +## Solution + +A short statement about how this PR solves the **Problem**. + +## Issues Resolved + +What issues does this PR resolve, if any? Please include lines like the following (i.e. "Resolves #NNNN), so that when this PR gets merged, GitHub will automatically close those issues. + +Resolves #NNNN +Resolves #MMMM + +## BEPs Implemented + +What [BEPs](https://github.com/bigchaindb/beps) does this pull request implement, if any? diff --git a/.gitignore b/.gitignore index b6e4761..1e0fb31 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,24 @@ __pycache__/ # C extensions *.so +# Swap -- copypasta from https://github.com/github/gitignore/blob/master/Global/Vim.gitignore +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags + # Distribution / packaging .Python +env/ build/ develop-eggs/ dist/ @@ -19,13 +35,9 @@ lib64/ parts/ sdist/ var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ *.egg-info/ .installed.cfg *.egg -MANIFEST # PyInstaller # Usually these files are written by a python script from a template @@ -40,16 +52,14 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ -.nox/ .coverage .coverage.* .cache +.pytest_cache/ nosetests.xml coverage.xml *.cover -*.py,cover .hypothesis/ -.pytest_cache/ # Translations *.mo @@ -58,72 +68,32 @@ coverage.xml # Django stuff: *.log local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy # Sphinx documentation -docs/_build/ +docs/build/ # PyBuilder target/ -# Jupyter Notebook +# Ipython Notebook .ipynb_checkpoints -# IPython -profile_default/ -ipython_config.py - # pyenv .python-version -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock +# Just in time documentation +docs/server/source/http-samples -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ +# Terraform state files +# See https://stackoverflow.com/a/41482391 +terraform.tfstate +terraform.tfstate.backup -# Celery stuff -celerybeat-schedule -celerybeat.pid +# tendermint data +tmdata/data +network/*/data -# SageMath parsed files -*.sage.py +# Docs that are fetched at build time +docs/contributing/source/cross-project-policies/*.md -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..32ba096 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +repos: +- repo: git://github.com/pre-commit/pre-commit-hooks + sha: v1.1.1 + hooks: + - id: trailing-whitespace + args: ['--no-markdown-linebreak-ext'] + - id: check-merge-conflict + - id: debug-statements + - id: check-added-large-files + - id: flake8 + +- repo: git://github.com/chewse/pre-commit-mirrors-pydocstyle + sha: v2.1.1 + hooks: + - id: pydocstyle + # list of error codes to check, see: http://www.pydocstyle.org/en/latest/error_codes.html + args: ['--select=D204,D201,D209,D210,D212,D300,D403'] + +# negate the exclude to only apply the hooks to 'bigchaindb' and 'tests' folder +exclude: '^(?!bigchaindb/)(?!tests/)(?!acceptance/)' diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..15aeb2c --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,11 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +build: + image: latest + +python: + version: 3.6 + pip_install: true diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e94eb1c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,77 @@ +# Copyright © 2020, 2021 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +sudo: required + +dist: focal + +services: + - docker + +language: python +cache: pip + +python: + - 3.6 + - 3.7 + - 3.8 + +env: + global: + - DOCKER_COMPOSE_VERSION=1.29.2 + matrix: + - TOXENV=flake8 + - TOXENV=docsroot + +matrix: + fast_finish: true + include: + - python: 3.6 + env: + - PLANETMINT_DATABASE_BACKEND=localmongodb + - PLANETMINT_DATABASE_SSL= + - python: 3.6 + env: + - PLANETMINT_DATABASE_BACKEND=localmongodb + - PLANETMINT_DATABASE_SSL= + - PLANETMINT_CI_ABCI=enable + - python: 3.6 + env: + - PLANETMINT_ACCEPTANCE_TEST=enable + - python: 3.7 + env: + - PLANETMINT_DATABASE_BACKEND=localmongodb + - PLANETMINT_DATABASE_SSL= + - python: 3.7 + env: + - PLANETMINT_DATABASE_BACKEND=localmongodb + - PLANETMINT_DATABASE_SSL= + - PLANETMINT_CI_ABCI=enable + - python: 3.7 + env: + - PLANETMINT_ACCEPTANCE_TEST=enable + - python: 3.8 + env: + - PLANETMINT_DATABASE_BACKEND=localmongodb + - PLANETMINT_DATABASE_SSL= + - python: 3.8 + env: + - PLANETMINT_DATABASE_BACKEND=localmongodb + - PLANETMINT_DATABASE_SSL= + - PLANETMINT_CI_ABCI=enable + - python: 3.8 + env: + - PLANETMINT_ACCEPTANCE_TEST=enable + + +before_install: sudo .ci/travis-before-install.sh + +install: .ci/travis-install.sh + +before_script: .ci/travis-before-script.sh + +script: .ci/travis_script.sh + +after_success: .ci/travis-after-success.sh diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ad3583a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,1162 @@ + + +# Change Log (Release Notes) + +All _notable_ changes to this project will be documented in this file (`CHANGELOG.md`). + +This project adheres to [the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) (or at least we try). The Planetmint public API _was_ defined in this file but that definition was moved and can now be found in [BEP-7](https://github.com/bigchaindb/BEPs/tree/master/7). + +Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/). Note that each version (or "release") is the name of a [Git _tag_](https://git-scm.com/book/en/v2/Git-Basics-Tagging) of a particular commit, so the associated date and time are the date and time of that commit (as reported by GitHub), _not_ the "Uploaded on" date listed on PyPI (which may differ). + +For reference, the possible headings are: + +* **Added** for new features. +* **Changed** for changes in existing functionality. +* **Deprecated** for once-stable features removed in upcoming releases. +* **Removed** for deprecated features removed in this release. +* **Fixed** for any bug fixes. +* **Security** to invite users to upgrade in case of vulnerabilities. +* **External Contributors** to list contributors outside of Planetmint GmbH. +* **Known Issues** +* **Notes** + +## [2.2.2] - 2020-08-12 + +### Security + +Several dependencies updated including Flask that had vulnerability. + +### Fixed + +* Updated priv_validator key format in stack script (#2707) + +### External Contributors + +* @aostrun - [#2708](https://github.com/bigchaindb/bigchaindb/pull/2708) + +## [2.2.1] - 2020-04-14 + +### Fixed + +Gevent library API update is incompatible with bigchaindb-abci 1.0.1 version. +Updated bigchaindb-abci. + +## [2.2.0] - 2020-02-20 + +### Added + +Support for multiple ABCI versions. + +## [2.1.0] - 2019-11-06 + +### Added + +Option for last transaction retrieval added. + +## [2.0] - 2019-09-26 + +### Changed + +Migrated from Tendermint 0.22.8 to 0.31.5. + +## [2.0 Beta 9] - 2018-11-27 + +### Changed + +Removed support for TLSv1 and TLSv1.1 in all NGINX config files. Kept support for TLSv1.2 and added support for TLSv1.3. [Pull Request #2601](https://github.com/bigchaindb/bigchaindb/pull/2601) + +### Fixed + +Fixed two issues with schema validation. Pull requests [#2606](https://github.com/bigchaindb/bigchaindb/pull/2606) & [#2607](https://github.com/bigchaindb/bigchaindb/pull/2607) + +### External Contributors + +[@gamjapark](https://github.com/gamjapark) and team translated all the [Planetmint root docs](https://docs.bigchaindb.com/en/latest/korean/index.html) into Korean. [Pull Request #2603](https://github.com/bigchaindb/bigchaindb/pull/2603) + +## [2.0 Beta 8] - 2018-11-03 + +### Changed + +* Revised the [Simple Deployment Template](http://docs.bigchaindb.com/projects/server/en/latest/simple-deployment-template/index.html) in the docs. Added NGINX to the mix. Pull Requests [#2578](https://github.com/bigchaindb/bigchaindb/pull/2578) and [#2579](https://github.com/bigchaindb/bigchaindb/pull/2579) +* Revised `nginx/nginx.conf` to enable CORS. [Pull Request #2580](https://github.com/bigchaindb/bigchaindb/pull/2580) + +### Fixed + +* Fixed a typo in the Kubernetes ConfigMap template. [Pull Request #2583](https://github.com/bigchaindb/bigchaindb/pull/2583) + +### External Contributors + +[@gamjapark](https://github.com/gamjapark) translated the main `README.md` file into Korean. [Pull Request #2592](https://github.com/bigchaindb/bigchaindb/pull/2592) + +## [2.0 Beta 7] - 2018-09-28 + +Tag name: v2.0.0b7 + +### Added + +Completed the implementation of chain-migration elections (BEP-42). Pull requests [#2553](https://github.com/bigchaindb/bigchaindb/pull/2553), [#2556](https://github.com/bigchaindb/bigchaindb/pull/2556), [#2558](https://github.com/bigchaindb/bigchaindb/pull/2558), [#2563](https://github.com/bigchaindb/bigchaindb/pull/2563) and [#2566](https://github.com/bigchaindb/bigchaindb/pull/2566) + +### Changed + +* Code that used the Python driver's (deprecated) transactions.send() method now uses its transactions.send_commit() method instead. [Pull request #2547](https://github.com/bigchaindb/bigchaindb/pull/2547) +* Code that implied pluggable "consensus" now implies pluggable transaction "validation" (a more accurate word). [Pull request #2561](https://github.com/bigchaindb/bigchaindb/pull/2561) + +### Removed + +Benchmark logs. [Pull request #2565](https://github.com/bigchaindb/bigchaindb/pull/2565) + +### Fixed + +A bug caused by an incorrect MongoDB query. [Pull request #2567](https://github.com/bigchaindb/bigchaindb/pull/2567) + +### Notes + +There's now better documentation about logs, log rotation, and the `server.bind` config setting. Pull requests [#2546](https://github.com/bigchaindb/bigchaindb/pull/2546) and [#2575](https://github.com/bigchaindb/bigchaindb/pull/2575) + +## [2.0 Beta 6] - 2018-09-17 + +Tag name: v2.0.0b6 + +### Added + +* [New documentation about privacy and handling private data](https://docs.bigchaindb.com/en/latest/private-data.html). [Pull request #2437](https://github.com/bigchaindb/bigchaindb/pull/2437) +* New documentation about log rotation. Also rotate Tendermint logs if started using Monit. [Pull request #2528](https://github.com/bigchaindb/bigchaindb/pull/2528) +* Began implementing one of the migration strategies outlined in [BEP-42](https://github.com/bigchaindb/BEPs/tree/master/42). That involved creating a more general-purpose election process and commands. Pull requests [#2488](https://github.com/bigchaindb/bigchaindb/pull/2488), [#2495](https://github.com/bigchaindb/bigchaindb/pull/2495), [#2498](https://github.com/bigchaindb/bigchaindb/pull/2498), [#2515](https://github.com/bigchaindb/bigchaindb/pull/2515), [#2535](https://github.com/bigchaindb/bigchaindb/pull/2535) +* Used memoization to avoid doing some validation checks multiple times. [Pull request #2490](https://github.com/bigchaindb/bigchaindb/pull/2490) +* Created an all-in-one Docker image containing Planetmint Server, Tendermint and MongoDB. It was created for a particular user and is not recommended for production use unless you really know what you're doing. [Pull request #2424](https://github.com/bigchaindb/bigchaindb/pull/2424) + +### Changed + +* The supported versions of Tendermint are now hard-wired into Planetmint Server: it checks to see what version the connected Tendermint has, and if it's not compatible, Planetmint Server exits with an error message. [Pull request #2541](https://github.com/bigchaindb/bigchaindb/pull/2541) +* The docs no longer say to install the highest version of Tendermint: they say to install a specific version. [Pull request #2524](https://github.com/bigchaindb/bigchaindb/pull/2524) +* The setup docs include more recommended settings for `config.toml`. [Pull request #2516](https://github.com/bigchaindb/bigchaindb/pull/2516) +* The process to add, remove or update the voting power of a validator at run time (using the `bigchaindb upsert-validator` subcommands) was completely changed and is now fully working. See [issue #2372](https://github.com/bigchaindb/bigchaindb/issues/2372) and all the pull requests it references. Pull requests [#2439](https://github.com/bigchaindb/bigchaindb/pull/2439) and [#2440](https://github.com/bigchaindb/bigchaindb/pull/2440) +* The license on the documentation was changed from CC-BY-SA-4 to CC-BY-4. [Pull request #2427](https://github.com/bigchaindb/bigchaindb/pull/2427) +* Re-activated and/or updated some unit tests that had been deacivated during the migration to Tendermint. Pull requests [#2390](https://github.com/bigchaindb/bigchaindb/pull/2390), [#2415](https://github.com/bigchaindb/bigchaindb/pull/2415), [#2452](https://github.com/bigchaindb/bigchaindb/pull/24), [#2456](https://github.com/bigchaindb/bigchaindb/pull/2456) +* Updated RapidJSON to a newer, faster version. [Pull request #2470](https://github.com/bigchaindb/bigchaindb/pull/2470) +* The Java driver is now officially supported. [Pull request #2478](https://github.com/bigchaindb/bigchaindb/pull/2478) +* The MongoDB indexes on transaction id and block height were changed to be [unique indexes](https://docs.mongodb.com/manual/core/index-unique/). [Pull request #2492](https://github.com/bigchaindb/bigchaindb/pull/2492) +* Updated the required `cryptoconditions` package to a newer one. [Pull request #2494](https://github.com/bigchaindb/bigchaindb/pull/2494) + +### Removed + +* Removed some old code and tests. Pull requests + [#2374](https://github.com/bigchaindb/bigchaindb/pull/2374), + [#2452](https://github.com/bigchaindb/bigchaindb/pull/2452), + [#2474](https://github.com/bigchaindb/bigchaindb/pull/2474), + [#2476](https://github.com/bigchaindb/bigchaindb/pull/2476), + [#2491](https://github.com/bigchaindb/bigchaindb/pull/2491) + +### Fixed + +* Fixed the Events API so that it only sends valid transactions to subscribers. Also changed how it works internally, so now it is more reliable. [Pull request #2529](https://github.com/bigchaindb/bigchaindb/pull/2529) +* Fixed a bug where MongoDB database initialization would abort if a collection already existed. [Pull request #2520](https://github.com/bigchaindb/bigchaindb/pull/2520) +* Fixed a unit test that was failing randomly. [Pull request #2423](https://github.com/bigchaindb/bigchaindb/pull/2423) +* Fixed the validator curl port. [Pull request #2447](https://github.com/bigchaindb/bigchaindb/pull/2447) +* Fixed an error in the docs about the HTTP POST /transactions endpoint. [Pull request #2481](https://github.com/bigchaindb/bigchaindb/pull/2481) +* Fixed a unit test that could loop forever. [Pull requqest #2486](https://github.com/bigchaindb/bigchaindb/pull/2486) +* Fixed a bug when validating a CREATE + TRANSFER. [Pull request #2487](https://github.com/bigchaindb/bigchaindb/pull/2487) +* Fixed the HTTP response when posting a transaction in commit mode. [Pull request #2510](https://github.com/bigchaindb/bigchaindb/pull/2510) +* Fixed a crash that happened when attempting to restart Planetmint at Tendermint block height 1. [Pull request#2519](https://github.com/bigchaindb/bigchaindb/pull/2519) + +### External Contributors + +@danacr - [Pull request #2447](https://github.com/bigchaindb/bigchaindb/pull/2447) + +### Notes + +The docs section titled "Production Deployment Template" was renamed to "Kubernetes Deployment Template" and we no longer consider it the go-to deployment template. The "Simple Deployment Template" is simpler, easier to understand, and less expensive (unless you are with an organization that already has a big Kubernetes cluster). + +## [2.0 Beta 5] - 2018-08-01 + +Tag name: v2.0.0b5 + +### Changed + +* Supported version of Tendermint `0.22.3` -> `0.22.8`. [Pull request #2429](https://github.com/bigchaindb/bigchaindb/pull/2429). + +### Fixed + +* Stateful validation raises a DoubleSpend exception if there is any other transaction that spends the same output(s) even if it has the same transaction ID. [Pull request #2422](https://github.com/bigchaindb/bigchaindb/pull/2422). + +## [2.0 Beta 4] - 2018-07-30 + +Tag name: v2.0.0b4 + +### Added + +- Added scripts for creating a configuration to manage processes with Monit. [Pull request #2410](https://github.com/bigchaindb/bigchaindb/pull/2410). + +### Fixed + +- Redundant asset and metadata queries were removed. [Pull request #2409](https://github.com/bigchaindb/bigchaindb/pull/2409). +- Signal handling was fixed for Planetmint processes. [Pull request #2395](https://github.com/bigchaindb/bigchaindb/pull/2395). +- Some of the abruptly closed sockets that used to stay in memory are being cleaned up now. [Pull request 2408](https://github.com/bigchaindb/bigchaindb/pull/2408). +- Fixed the bug when WebSockets powering Events API became unresponsive. [Pull request #2413](https://github.com/bigchaindb/bigchaindb/pull/2413). + +### Notes: + +* The instructions on how to write a BEP were simplified. [Pull request #2347](https://github.com/bigchaindb/bigchaindb/pull/2347). +* A section about troubleshooting was added to the network setup guide. [Pull request #2398](https://github.com/bigchaindb/bigchaindb/pull/2398). +* Some of the core code was given a better package structure. [Pull request #2401](https://github.com/bigchaindb/bigchaindb/pull/2401). +* Some of the previously disabled unit tests were re-enabled and updated. Pull requests [#2404](https://github.com/bigchaindb/bigchaindb/pull/2404) and [#2402](https://github.com/bigchaindb/bigchaindb/pull/2402). +* Some building blocks for dynamically adding new validators were introduced. [Pull request #2392](https://github.com/bigchaindb/bigchaindb/pull/2392). + +## [2.0 Beta 3] - 2018-07-18 + +Tag name: v2.0.0b3 + +### Fixed + +Fixed a bug in transaction validation. For some more-complex situations, it would say that a valid transaction was invalid. This bug was actually fixed before; it was [issue #1271](https://github.com/bigchaindb/bigchaindb/issues/1271). The unit test for it was turned off while we integrated Tendermint. Then the query implementation code got changed, reintroducing the bug, but the unit test was off so the bug wasn't caught. When we turned the test back on, shortly after releasing Beta 2, it failed, unveiling the bug. [Pull request #2389](https://github.com/bigchaindb/bigchaindb/pull/2389) + +## [2.0 Beta 2] - 2018-07-16 + +Tag name: v2.0.0b2 + +### Added + +* Added new configuration settings `tendermint.host` and `tendermint.port`. [Pull request #2342](https://github.com/bigchaindb/bigchaindb/pull/2342) +* Added tests to ensure that Planetmint gracefully handles "nasty" strings in keys and values. [Pull request #2334](https://github.com/bigchaindb/bigchaindb/pull/2334) +* Added a new logging handler to capture benchmark stats to a separate file. [Pull request #2349](https://github.com/bigchaindb/bigchaindb/pull/2349) + +### Changed + +* Changed the names of Planetmint processes (Python processes) to include 'bigchaindb', so they are easier to spot and find. [Pull request #2354](https://github.com/bigchaindb/bigchaindb/pull/2354) +* Updated all code to support the latest version of Tendermint. Note that the Planetmint ABCI server now listens to port 26657 instead of 46657. Pull requests [#2375](https://github.com/bigchaindb/bigchaindb/pull/2375) and [#2380](https://github.com/bigchaindb/bigchaindb/pull/2380) + +### Removed + +Removed all support and code for the old backlog_reassign_delay setting. [Pull request #2332](https://github.com/bigchaindb/bigchaindb/pull/2332) + +### Fixed + +* Fixed a bug that sometimes arose when using Docker Compose. (Tendermint would freeze.) [Pull request #2341](https://github.com/bigchaindb/bigchaindb/pull/2341) +* Fixed a bug in the code that creates a MongoDB index for the "id" in the transactions collection. It works now, and performance is improved. [Pull request #2378](https://github.com/bigchaindb/bigchaindb/pull/2378) +* The logging server would keep runnning in some tear-down scenarios. It doesn't do that any more. [Pull request #2304](https://github.com/bigchaindb/bigchaindb/pull/2304) + +### External Contributors + +@hrntknr - [Pull request #2331](https://github.com/bigchaindb/bigchaindb/pull/2331) + +### Known Issues + +The `bigchaindb upsert-validator` subcommand is not working yet, but a solution ([BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21)) has been finalized and will be implemented before we release the final Planetmint 2.0. + +### Notes + +* A lot of old/dead code was deleted. Pull requests +[#2319](https://github.com/bigchaindb/bigchaindb/pull/2319), +[#2338](https://github.com/bigchaindb/bigchaindb/pull/2338), +[#2357](https://github.com/bigchaindb/bigchaindb/pull/2357), +[#2365](https://github.com/bigchaindb/bigchaindb/pull/2365), +[#2366](https://github.com/bigchaindb/bigchaindb/pull/2366), +[#2368](https://github.com/bigchaindb/bigchaindb/pull/2368) and +[#2374](https://github.com/bigchaindb/bigchaindb/pull/2374) +* Improved the documentation page "How to setup a Planetmint Network". [Pull Request #2312](https://github.com/bigchaindb/bigchaindb/pull/2312) + +## [2.0 Beta 1] - 2018-06-01 + +Tag name: v2.0.0b1 + +### Fixed + +* Fixed a bug that arose with some code that treated transactions-waiting-for-block-inclusion as if they were already stored in MongoDB (i.e. already in a block). [Pull request #2318](https://github.com/bigchaindb/bigchaindb/pull/2318) +* If a user asked for a block and it happened to be an empty block, Planetmint returned 404 Not Found, as if the block did not exist. Now it returns a 200 OK with a block containing no transactions, i.e. an empty block. [Pull request #2321](https://github.com/bigchaindb/bigchaindb/pull/2321) + +### Known Issues + +* An issue was found with the `bigchaindb upsert-validator` command. A solution was proposed in [BEP-19](https://github.com/bigchaindb/BEPs/pull/45) and is being implemented in [pull request #2314](https://github.com/bigchaindb/bigchaindb/pull/2314) +* If you run Planetmint locally using `make start` (i.e. using Docker Compose) and then you put the node under heavy write load, Tendermint can become unresponsive and running `make stop` can hang. +* There seems to be one or more issues with Tendermint when it is put under heavy load (i.e. even when Planetmint isn't involved). See Tendermint issues [#1394](https://github.com/tendermint/tendermint/issues/1394), [#1642](https://github.com/tendermint/tendermint/issues/1642) and [#1661](https://github.com/tendermint/tendermint/issues/1661) + +### Notes + +* There's a [new docs page](https://docs.bigchaindb.com/projects/server/en/v2.0.0b1/simple-network-setup.html) about how to set up a network where each node uses a single virtual machine. +* We checked, and Planetmint 2.0 Beta 1 works with MongoDB 3.6 (and 3.4). +* Support for RethinkDB is completely gone. + +## [2.0 Alpha 6] - 2018-05-17 + +Tag name: v2.0.0a6 + +### Changed + +Upgraded PyMongo to version 3.6 (which is compatible with MongoDB 3.6, 3.4 [and more](https://docs.mongodb.com/ecosystem/drivers/driver-compatibility-reference/#python-driver-compatibility)). [Pull request #2298](https://github.com/bigchaindb/bigchaindb/pull/2298) + +### Fixed + +When deploying a node using our Docker Compose file, it didn't expose port 46656, which is used by Tendermint for inter-node communications, so the node couldn't communicate with other nodes. We fixed that in [pull request #2299](https://github.com/bigchaindb/bigchaindb/pull/2299) + +### Notes + +We ran all our tests using MongoDB 3.6 and they all passed, so it seems safe to use Planetmint with MongoDB 3.6 from now on. + +## [2.0 Alpha 5] - 2018-05-11 + +Tag name: v2.0.0a5 + +### Changed + +To resolve [issue #2279](https://github.com/bigchaindb/bigchaindb/issues/2279), we made some changes to the `bigchaindb-abci` package (which is our fork of `py-abci`) and told Planetmint to use the new version (`bigchaindb-abci==0.4.5`). [Pull request #2281](https://github.com/bigchaindb/bigchaindb/pull/2281). + +## [2.0 Alpha 4] - 2018-05-09 + +Tag name: v2.0.0a4 + +### Changed + +The Kubernetes liveness probe for the Planetmint StatefulSet was improved to check the Tendermint /status endpoint in addition to the Tendermint /abci_info endpoint. [Pull request #2275](https://github.com/bigchaindb/bigchaindb/pull/2275) + +### Fixed + +[Pull request #2270](https://github.com/bigchaindb/bigchaindb/pull/2270) resolved [issue #2269](https://github.com/bigchaindb/bigchaindb/issues/2269). + +### Notes + +There's a new [page in the docs about storing files in Planetmint](https://docs.bigchaindb.com/en/latest/store-files.html). [Pull request #2259](https://github.com/bigchaindb/bigchaindb/pull/2259) + +## [2.0 Alpha 3] - 2018-05-03 + +Tag name: v2.0.0a3 + +### Changed + +* Upgraded Planetmint Server code to use the latest version of Tendermint: version 0.19.2. Pull requests [#2249](https://github.com/bigchaindb/bigchaindb/pull/2249), [#2252](https://github.com/bigchaindb/bigchaindb/pull/2252) and [#2253](https://github.com/bigchaindb/bigchaindb/pull/2253) +* Made some fixes to `py-abci` (an external Python package) and used our fixed version with Planetmint. Those fixes resolved several known issues, including [issue #2182](https://github.com/bigchaindb/bigchaindb/issues/2182) and issues with large transactions in general. Note: At the time of writing, our fixes to `py-abci` hadn't been merged into the main `py-abci` repository or its latest release on PyPI; we were using our own special `bigchaindb-abci` package (which included our fixes). Pull requests [#2250](https://github.com/bigchaindb/bigchaindb/pull/2250) and [#2261](https://github.com/bigchaindb/bigchaindb/pull/2261) +* If Planetmint Server crashes and then comes back, Tendermint Core doesn't try to reconnect to it. That's just how Tendermint Core works. We revised our Kubernetes-based production deployment template to resolve that issue: Planetmint Server and Tendermint Core are now in the same Kubernetes StatefulSet; if the connection between them ever goes down, then Kubernetes restarts the whole StatefulSet. [Pull request #2242](https://github.com/bigchaindb/bigchaindb/pull/2242) + +### Fixed + +Re-enabled multi-threading. [Pull request #2258](https://github.com/bigchaindb/bigchaindb/pull/2258) + +### Known Issues + +Tendermint changed how it responds to a request to store data (via the [Tendermint Broadcast API](https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api)) between version 0.12 and 0.19.2. We started modifying the code of Planetmint Server to account for those changes in responses (in [pull request #2239](https://github.com/bigchaindb/bigchaindb/pull/2239)), but we found that there's a difference between what the Tendermint documentation _says_ about those responses and how Tendermint actually responds. We need to determine Tendermint's intent before we can finalize that pull request. + +### Notes + +We were focused on getting the public Planetmint Testnet stable during the development of Planetmint 2.0 Alpha 3, and we think we largely succeeded. Because of that focus, we delayed the deployment of an internal test network until later. It would have had the same instabilities as the public Planetmint Testnet anyway. In the future, we'll always test a new version of Planetmint on our internal test network before deploying it on the public Planetmint Testnet. (That wasn't possible this time around, because there was no old/stable version of Planetmint 2.n to run on the public Planetmint Testnet while we tested Planetmint 2.[n+1] internally.) + +## [2.0 Alpha 2] - 2018-04-18 + +Tag name: v2.0.0a2 + +### Added + +An implementation of [BEP-8 (Planetmint Enhancement Proposal #8)](https://github.com/bigchaindb/BEPs/tree/master/8), which makes sure a node can recover from a system fault (e.g. a crash) into a consistent state, i.e. a state where the data in the node's local MongoDB database is consistent with the data stored in the blockchain. Pull requests [#2135](https://github.com/bigchaindb/bigchaindb/pull/2135) and [#2207](https://github.com/bigchaindb/bigchaindb/pull/2207) + +### Changed + +When someone uses the HTTP API to send a new transaction to a Planetmint network using the [POST /api/v1/transactions?mode={mode}](https://docs.bigchaindb.com/projects/server/en/master/http-client-server-api.html#post--api-v1-transactions?mode=mode) endpoint, they now get back a more informative HTTP response, so they can better-understand what happened. This is only when mode is `commit` or `sync`, because `async` _means_ that the response is immediate, without waiting to see what happened to the transaction. [Pull request #2198](https://github.com/bigchaindb/bigchaindb/pull/2198) + +### Known Issues + +* If Planetmint Server crashes and then is restarted, Tendermint Core won't try to reconnect to Planetmint Server and so all operations requiring that connection won't work. We only understood this recently. We'll write a blog post explaining what we intend to do about it. +* The known issues in 2.0 Alpha (listed below) are still there. + +## [2.0 Alpha] - 2018-04-03 + +Tag name: v2.0.0a1 + +There were _many_ changes between 1.3 and 2.0 Alpha, too many to list here. We wrote a series of blog posts to summarize most changes, especially those that affect end users and application developers: + +* [Planetmint 2.0 is Byzantine Fault Tolerant](https://blog.bigchaindb.com/bigchaindb-2-0-is-byzantine-fault-tolerant-5ffdac96bc44) +* [Some HTTP API Changes in the Next Release](https://blog.bigchaindb.com/some-http-api-changes-in-the-next-release-49612a537b0c) +* [Three Transaction Model Changes in the Next Release](https://blog.bigchaindb.com/three-transaction-model-changes-in-the-next-release-dadbac50094a) +* [Changes to the Server Command Line Interface in Planetmint 2.0](https://blog.bigchaindb.com/changes-to-the-server-command-line-interface-in-bigchaindb-2-0-e1d6576e7155) +* _A forthcoming post about changes in Planetmint Server configuration settings_ + +### External Contributors + +* @r7vme contributed [pull request #1984](https://github.com/bigchaindb/bigchaindb/pull/1984) which fixed a bug in our Kubernetes-based production deployment template. + +### Known Issues + +We intend to resolve these issues before releasing the final Planetmint 2.0: + +* There's a known Heisenbug that (sometimes) arises and we found that making the Planetmint webserver single-threaded prevents that bug from causing problems. We intend to resolve that bug, but in the meantime our temporary workaround is to change the _default_ webserver configuration settings to single-threaded mode, i.e. `PLANETMINT_SERVER_WORKERS=1` and `PLANETMINT_SERVER_THREADS=1`. +* Issues sometimes happen when a large transaction is sent to a Planetmint network. + +## [1.3] - 2017-11-21 +Tag name: v1.3.0 + +### Added +* Metadata full-text search. [Pull request #1812](https://github.com/bigchaindb/bigchaindb/pull/1812) + +### Notes +* Improved documentation about blocks and votes. [Pull request #1855](https://github.com/bigchaindb/bigchaindb/pull/1855) + + +## [1.2] - 2017-11-13 +Tag name: v1.2.0 + +### Added +* New and improved installation setup docs and code. Pull requests [#1775](https://github.com/bigchaindb/bigchaindb/pull/1775) and [#1785](https://github.com/bigchaindb/bigchaindb/pull/1785) +* New Planetmint configuration setting to set the port number of the log server: `log.port`. [Pull request #1796](https://github.com/bigchaindb/bigchaindb/pull/1796) +* New secondary index on `id` in the bigchain table. That will make some queries execute faster. [Pull request #1803](https://github.com/bigchaindb/bigchaindb/pull/1803) +* When using MongoDB, there are some restrictions on allowed names for keys (JSON keys). Those restrictions were always there but now Planetmint checks key names explicitly, rather than leaving that to MongoDB. Pull requests [#1807](https://github.com/bigchaindb/bigchaindb/pull/1807) and [#1811](https://github.com/bigchaindb/bigchaindb/pull/1811) +* When using MongoDB, there are some restrictions on the allowed values of "language" (if that key is used in the values of `metadata` or `asset.data`). Those restrictions were always there but now Planetmint checks the values explicitly, rather than leaving that to MongoDB. Pull requests [#1806](https://github.com/bigchaindb/bigchaindb/pull/1806) and [#1811](https://github.com/bigchaindb/bigchaindb/pull/1811) +* There's a new page in the root docs about permissions in Planetmint. [Pull request #1788](https://github.com/bigchaindb/bigchaindb/pull/1788) +* There's a new option in the `bigchaindb start` command: `bigchaindb start --no-init` will avoid doing `bigchaindb init` if it wasn't done already. [Pull request #1814](https://github.com/bigchaindb/bigchaindb/pull/1814) + +### Fixed +* Fixed a bug where setting the log level in a Planetmint config file didn't have any effect. It does now. [Pull request #1797](https://github.com/bigchaindb/bigchaindb/pull/1797) +* The docs were wrong about there being no Ping/Pong support in the Events API. There is, so the docs were fixed. [Pull request #1799](https://github.com/bigchaindb/bigchaindb/pull/1799) +* Fixed an issue with closing WebSocket connections properly. [Pull request #1819](https://github.com/bigchaindb/bigchaindb/pull/1819) + +### Notes +* Many changes were made to the Kubernetes-based production deployment template and code. + + +## [1.1] - 2017-09-26 +Tag name: v1.1.0 + +### Added +* Support for server-side plugins that can add support for alternate event consumers (other than the WebSocket API). [Pull request #1707](https://github.com/bigchaindb/bigchaindb/pull/1707) +* New configuration settings to set the *advertised* wsserver scheme, host and port. (The *advertised* ones are the ones that external users use to connect to the WebSocket API.) [Pull request #1703](https://github.com/bigchaindb/bigchaindb/pull/1703) +* Support for secure (TLS) WebSocket connections. [Pull request #1619](https://github.com/bigchaindb/bigchaindb/pull/1619) +* A new page of documentation about the contents of a condition (inside a transaction). [Pull request #1668](https://github.com/bigchaindb/bigchaindb/pull/1668) + +### Changed +* We updated our definition of the **public API** (at the top of this document). [Pull request #1700](https://github.com/bigchaindb/bigchaindb/pull/1700) +* The HTTP API Logger now logs the request path and method as well. [Pull request #1644](https://github.com/bigchaindb/bigchaindb/pull/1644) + +### External Contributors +* @carchrae - [Pull request #1731](https://github.com/bigchaindb/bigchaindb/pull/1731) +* @ivanbakel - [Pull request #1706](https://github.com/bigchaindb/bigchaindb/pull/1706) +* @ketanbhatt - Pull requests [#1643](https://github.com/bigchaindb/bigchaindb/pull/1643) and [#1644](https://github.com/bigchaindb/bigchaindb/pull/1644) + +### Notes +* New drivers & tools from our community: + * [Java driver](https://github.com/authenteq/java-bigchaindb-driver), by [Authenteq](https://authenteq.com/) + * [Ruby library](https://rubygems.org/gems/bigchaindb), by @nileshtrivedi +* Many improvements to our production deployment template (which uses Kubernetes). +* The production deployment template for the multi-node case was out of date. We updated that and verified it. [Pull request #1713](https://github.com/bigchaindb/bigchaindb/pull/1713) + + +## [1.0.1] - 2017-07-13 +Tag name: v1.0.1 + +### Fixed +* Various issues in the Quickstart page. Pull requests + [#1641](https://github.com/bigchaindb/bigchaindb/pull/1641) and + [#1648](https://github.com/bigchaindb/bigchaindb/pull/1648). +* Changefeed hanging when MongoDB primary node is turned off. + [Pull request #1638](https://github.com/bigchaindb/bigchaindb/pull/1638). +* Missing `assets` tables for RethinkDB backend. + [Pull request #1646](https://github.com/bigchaindb/bigchaindb/pull/1646). +* Cryptoconditions version mismatch. + [Pull request #1659](https://github.com/bigchaindb/bigchaindb/pull/1659). + + +## [1.0.0] - 2017-07-05 +Tag name: v1.0.0 + +**This just reports the changes since the release of 1.0.0rc1. If you want the full picture of all changes since 0.10, then read the 1.0.0rc1 change log below as well as the upgrade guide.** + +### Changed +* The file name of the upgrade guide changed from `docs/upgrade-guides/v0.10-->v1.0.md` to `docs/upgrade-guides/v0.10-v1.0.md`. +* In `transaction.inputs[n].fulfills`, `output` was renamed to `output_index`. [Pull Request #1596](https://github.com/bigchaindb/bigchaindb/pull/1596) +* In `transaction.outputs[n].condition.details`, 1) `signature` was removed (from signature conditions) and 2) `subfulfillments` was renamed to `subconditions` (in threshold conditions). [Pull Request #1589](https://github.com/bigchaindb/bigchaindb/pull/1589) +* Refined transaction schema validation to check that the `transaction.outputs[n].condition.uri` corresponds to a condition that Planetmint Server 1.0.0 actually supports. [Pull Request #1597](https://github.com/bigchaindb/bigchaindb/pull/1597) +* Before, GET requests (to the HTTP API) with header `Content-Type: 'application/json'` would get a response with the message, "The browser (or proxy) sent a request that this server could not understand." Now, if a GET request includes a `Content-Type` header, that header gets deleted (i.e. ignored). [Pull Request #1630](https://github.com/bigchaindb/bigchaindb/pull/1630) + +### Fixed +* If an end user sends a transaction with `operation` equal to `GENESIS`, it will be rejected as invalid. [Pull Request #1612](https://github.com/bigchaindb/bigchaindb/pull/1612) + + +## [1.0.0rc1] - 2017-06-23 +Tag name: v1.0.0rc1 + +### Added +* Support for secure TLS/SSL communication between MongoDB and {Planetmint, MongoDB Backup Agent, MongoDB Monitoring Agent}. Pull Requests +[#1456](https://github.com/bigchaindb/bigchaindb/pull/1456), +[#1497](https://github.com/bigchaindb/bigchaindb/pull/1497), +[#1510](https://github.com/bigchaindb/bigchaindb/pull/1510), +[#1536](https://github.com/bigchaindb/bigchaindb/pull/1536), +[#1551](https://github.com/bigchaindb/bigchaindb/pull/1551) and +[#1552](https://github.com/bigchaindb/bigchaindb/pull/1552). +* Text search support (only if using MongoDB). Pull Requests [#1469](https://github.com/bigchaindb/bigchaindb/pull/1469) and [#1471](https://github.com/bigchaindb/bigchaindb/pull/1471) +* The `database.connection_timeout` configuration setting now works with RethinkDB too. [#1512](https://github.com/bigchaindb/bigchaindb/pull/1512) +* New code and tools for benchmarking CREATE transactions. [Pull Request #1511](https://github.com/bigchaindb/bigchaindb/pull/1511) + +### Changed +* There's an upgrade guide in `docs/upgrade-guides/v0.10-->v1.0.md`. It only covers changes to the transaction model and HTTP API. If that file hasn't been merged yet, see [Pull Request #1547](https://github.com/bigchaindb/bigchaindb/pull/1547) +* Cryptographic signatures now sign the whole (serialized) transaction body, including the transaction ID, but with all `"fulfillment"` values changed to `None`. [Pull Request #1225](https://github.com/bigchaindb/bigchaindb/pull/1225) +* In transactions, the value of `"amount"` must be a string. (Before, it was supposed to be a number.) [Pull Request #1286](https://github.com/bigchaindb/bigchaindb/pull/1286) +* In `setup.py`, the "Development Status" (as reported on PyPI) was changed from Alpha to Beta. [Pull Request #1437](https://github.com/bigchaindb/bigchaindb/pull/1437) +* If you explicitly specify a config file, e.g. `bigchaindb -c path/to/config start` and that file can't be found, then Planetmint Server will fail with a helpful error message. [Pull Request #1486](https://github.com/bigchaindb/bigchaindb/pull/1486) +* Reduced the response time on the HTTP API endpoint to get all the unspent outputs associated with a given public key (a.k.a. "fast unspents"). [Pull Request #1411](https://github.com/bigchaindb/bigchaindb/pull/1411) +* Internally, the value of an asset's `"data"` is now stored in a separate assets table. This enabled the new text search. Each asset data is stored along with the associated CREATE transaction ID (asset ID). That data gets written when the containing block gets written to the bigchain table. [Pull Request #1460](https://github.com/bigchaindb/bigchaindb/pull/1460) +* Schema validation was sped up by switching to `rapidjson-schema`. [Pull Request #1494](https://github.com/bigchaindb/bigchaindb/pull/1494) +* If a node comes back from being down for a while, it will resume voting on blocks in the order determined by the MongoDB oplog, in the case of MongoDB. (In the case of RethinkDB, blocks missed in the changefeed will not be voted on.) [Pull Request #1389](https://github.com/bigchaindb/bigchaindb/pull/1389) +* Parallelized transaction schema validation in the vote pipeline. [Pull Request #1492](https://github.com/bigchaindb/bigchaindb/pull/1492) +* `asset.data` or `asset.id` are now *required* in a CREATE or TRANSFER transaction, respectively. [Pull Request #1518](https://github.com/bigchaindb/bigchaindb/pull/1518) +* The HTTP response body, in the response to the `GET /` and the `GET /api/v1` endpoints, was changed substantially. [Pull Request #1529](https://github.com/bigchaindb/bigchaindb/pull/1529) +* Changed the HTTP `GET /api/v1/transactions/{transaction_id}` endpoint. It now only returns the transaction if it's in a valid block. It also returns a new header with a relative link to a status monitor. [Pull Request #1543](https://github.com/bigchaindb/bigchaindb/pull/1543) +* All instances of `txid` and `tx_id` were replaced with `transaction_id`, in the transaction model and the HTTP API. [Pull Request #1532](https://github.com/bigchaindb/bigchaindb/pull/1532) +* The hostname and port were removed from all URLs in all HTTP API responses. [Pull Request #1538](https://github.com/bigchaindb/bigchaindb/pull/1538) +* Relative links were replaced with JSON objects in HTTP API responses. [Pull Request #1541](https://github.com/bigchaindb/bigchaindb/pull/1541) +* In the outputs endpoint of the HTTP API, the query parameter `unspent` was changed to `spent` (so no more double negatives). If that query parameter isn't included, then all outputs matching the specificed public key will be returned. If `spent=true`, then only the spent outputs will be returned. If `spent=false`, then only the unspent outputs will be returned. [Pull Request #1545](https://github.com/bigchaindb/bigchaindb/pull/1545) +* The supported crypto-conditions changed from version 01 of the crypto-conditions spec to version 02. [Pull Request #1562](https://github.com/bigchaindb/bigchaindb/pull/1562) +* The value of "version" inside a transaction must now be "1.0". (Before, it could be "0.anything".) [Pull Request #1574](https://github.com/bigchaindb/bigchaindb/pull/1574) + +### Removed +* The `server.threads` configuration setting (for the Gunicorn HTTP server) was removed from the default set of Planetmint configuration settings. [Pull Request #1488](https://github.com/bigchaindb/bigchaindb/pull/1488) + +### Fixed +* The `GET /api/v1/outputs` endpoint was failing for some transactions with threshold conditions. Fixed in [Pull Request #1450](https://github.com/bigchaindb/bigchaindb/pull/1450) + +### External Contributors +* @elopio - Pull Requests [#1415](https://github.com/bigchaindb/bigchaindb/pull/1415) and [#1491](https://github.com/bigchaindb/bigchaindb/pull/1491) +* @CsterKuroi - [Pull Request #1447](https://github.com/bigchaindb/bigchaindb/pull/1447) +* @tdsgit - [Pull Request #1512](https://github.com/bigchaindb/bigchaindb/pull/1512) +* @lavinasachdev3 - [Pull Request #1357](https://github.com/bigchaindb/bigchaindb/pull/1357) + +### Notes +* We dropped support for Python 3.4. [Pull Request #1564](https://github.com/bigchaindb/bigchaindb/pull/1564) +* There were many improvements to our Kubernetes-based production deployment template (and the associated documentaiton). +* There is now a [Planetmint Ruby driver](https://github.com/LicenseRocks/bigchaindb_ruby), created by @addywaddy at [license.rocks](https://github.com/bigchaindb/bigchaindb/pull/1437). +* The [Planetmint JavaScript driver](https://github.com/bigchaindb/js-bigchaindb-driver) was moved to a different GitHub repo and is now officially maintained by the Planetmint team. +* We continue to recommend using MongoDB. + +## [0.10.3] - 2017-06-29 +Tag name: v0.10.3 + +## Fixed +* Pin minor+ version of `cryptoconditions` to avoid upgrading to a non + compatible version. +[commit 97268a5](https://github.com/bigchaindb/bigchaindb/commit/97268a577bf27942a87d8eb838f4816165c84fd5) + +## [0.10.2] - 2017-05-16 +Tag name: v0.10.2 + +### Added +* Add Cross Origin Resource Sharing (CORS) support for the HTTP API. + [Commit 6cb7596](https://github.com/bigchaindb/bigchaindb/commit/6cb75960b05403c77bdae0fd327612482589efcb) + +### Fixed +* Fixed `streams_v1` API link in response to `GET /api/v1`. + [Pull Request #1466](https://github.com/bigchaindb/bigchaindb/pull/1466) +* Fixed mismatch between docs and implementation for `GET /blocks?status=` + endpoint. The `status` query parameter is now case insensitive. + [Pull Request #1464](https://github.com/bigchaindb/bigchaindb/pull/1464) + +## [0.10.1] - 2017-04-19 +Tag name: v0.10.1 + +### Added +* Documentation for the Planetmint settings `wsserver.host` and `wsserver.port`. [Pull Request #1408](https://github.com/bigchaindb/bigchaindb/pull/1408) + +### Fixed +* Fixed `Dockerfile`, which was failing to build. It now starts `FROM python:3.6` (instead of `FROM ubuntu:xenial`). [Pull Request #1410](https://github.com/bigchaindb/bigchaindb/pull/1410) +* Fixed the `Makefile` so that `release` depends on `dist`. [Pull Request #1405](https://github.com/bigchaindb/bigchaindb/pull/1405) + +## [0.10.0] - 2017-04-18 +Tag name: v0.10.0 + +### Added +* Improved logging. Added logging to file. Added `--log-level` option to `bigchaindb start` command. Added new logging configuration settings. Pull Requests +[#1285](https://github.com/bigchaindb/bigchaindb/pull/1285), +[#1307](https://github.com/bigchaindb/bigchaindb/pull/1307), +[#1324](https://github.com/bigchaindb/bigchaindb/pull/1324), +[#1326](https://github.com/bigchaindb/bigchaindb/pull/1326), +[#1327](https://github.com/bigchaindb/bigchaindb/pull/1327), +[#1330](https://github.com/bigchaindb/bigchaindb/pull/1330), +[#1365](https://github.com/bigchaindb/bigchaindb/pull/1365), +[#1394](https://github.com/bigchaindb/bigchaindb/pull/1394), +[#1396](https://github.com/bigchaindb/bigchaindb/pull/1396), +[#1398](https://github.com/bigchaindb/bigchaindb/pull/1398) and +[#1402](https://github.com/bigchaindb/bigchaindb/pull/1402) +* Events API using WebSocket protocol. Pull Requests +[#1086](https://github.com/bigchaindb/bigchaindb/pull/1086), +[#1347](https://github.com/bigchaindb/bigchaindb/pull/1347), +[#1349](https://github.com/bigchaindb/bigchaindb/pull/1349), +[#1356](https://github.com/bigchaindb/bigchaindb/pull/1356), +[#1368](https://github.com/bigchaindb/bigchaindb/pull/1368), +[#1401](https://github.com/bigchaindb/bigchaindb/pull/1401) and +[#1403](https://github.com/bigchaindb/bigchaindb/pull/1403) +* Initial support for using SSL with MongoDB (work in progress). Pull Requests +[#1299](https://github.com/bigchaindb/bigchaindb/pull/1299) and +[#1348](https://github.com/bigchaindb/bigchaindb/pull/1348) + +### Changed +* The main Planetmint Dockerfile (and its generated Docker image) now contains only Planetmint Server. (It used to contain both Planetmint Server and RethinkDB.) You must now run MongoDB or RethinkDB in a separate Docker container. [Pull Request #1174](https://github.com/bigchaindb/bigchaindb/pull/1174) +* Made separate schemas for CREATE and TRANSFER transactions. [Pull Request #1257](https://github.com/bigchaindb/bigchaindb/pull/1257) +* When signing transactions with threshold conditions, we now sign all subconditions for a public key. [Pull Request #1294](https://github.com/bigchaindb/bigchaindb/pull/1294) +* Many changes to the voting-related code, including how we validate votes and prevent duplicate votes by the same node. Pull Requests [#1215](https://github.com/bigchaindb/bigchaindb/pull/1215) and [#1258](https://github.com/bigchaindb/bigchaindb/pull/1258) + +### Removed +* Removed the `bigchaindb load` command. Pull Requests +[#1261](https://github.com/bigchaindb/bigchaindb/pull/1261), +[#1273](https://github.com/bigchaindb/bigchaindb/pull/1273) and +[#1301](https://github.com/bigchaindb/bigchaindb/pull/1301) +* Removed old `/speed-tests` and `/benchmarking-tests` directories. [Pull Request #1359](https://github.com/bigchaindb/bigchaindb/pull/1359) + +### Fixed +* Fixed the URL of the Planetmint docs returned by the HTTP API. [Pull Request #1178](https://github.com/bigchaindb/bigchaindb/pull/1178) +* Fixed the MongoDB changefeed: it wasn't reporting update operations. [Pull Request #1193](https://github.com/bigchaindb/bigchaindb/pull/1193) +* Fixed the block-creation process: it wasn't checking if the transaction was previously included in: + * a valid block. [Pull Request #1208](https://github.com/bigchaindb/bigchaindb/pull/1208) + * the block-under-construction. Pull Requests [#1237](https://github.com/bigchaindb/bigchaindb/issues/1237) and [#1377](https://github.com/bigchaindb/bigchaindb/issues/1377) + +### External Contributors +In alphabetical order by GitHub username: +* @anryko - [Pull Request #1277](https://github.com/bigchaindb/bigchaindb/pull/1277) +* @anujism - [Pull Request #1366](https://github.com/bigchaindb/bigchaindb/pull/1366) +* @jackric - [Pull Request #1365](https://github.com/bigchaindb/bigchaindb/pull/1365) +* @lavinasachdev3 - [Pull Request #1358](https://github.com/bigchaindb/bigchaindb/pull/1358) +* @morrme - [Pull Request #1340](https://github.com/bigchaindb/bigchaindb/pull/1340) +* @tomconte - [Pull Request #1299](https://github.com/bigchaindb/bigchaindb/pull/1299) +* @tymlez - Pull Requests [#1108](https://github.com/bigchaindb/bigchaindb/pull/1108) & [#1209](https://github.com/bigchaindb/bigchaindb/pull/1209) + +### Notes +* MongoDB is now the recommended database backend (not RethinkDB). +* There are some initial docs about how to deploy a Planetmint node on Kubernetes. It's work in progress. + + +## [0.9.5] - 2017-03-29 +Tag name: v0.9.5 + +### Fixed +Upgrade `python-rapidjson` to `0.0.11`(fixes #1350 - thanks to @ferOnti for +reporting). + +## [0.9.4] - 2017-03-16 +Tag name: v0.9.4 + +### Fixed +Fixed #1271 (false double spend error). Thanks to @jmduque for reporting the +problem along with a very detailed diagnosis and useful recommendations. + +## [0.9.3] - 2017-03-06 +Tag name: v0.9.3 + +### Fixed +Fixed HTTP API 500 error on `GET /outputs`: issues #1200 and #1231. + +## [0.9.2] - 2017-03-02 +Tag name: v0.9.2 + +### Fixed +Pin `python-rapidjson` library in `setup.py` to prevent `bigchaindb`'s +installation to fail due to +https://github.com/python-rapidjson/python-rapidjson/issues/62. + +## [0.9.1] - 2017-02-06 +Tag name: v0.9.1 + +### Fixed +* Fixed bug in how the transaction `VERSION` string was calculated from the Planetmint Server `__short_version__` string. [Pull Request #1160](https://github.com/bigchaindb/bigchaindb/pull/1160) + + +## [0.9.0] - 2017-02-06 +Tag name: v0.9.0 + +It has been more than two months since the v0.8.0 release, so there have been _many_ changes. We decided to describe them in broad strokes, with links to more details elsewhere. + +### Added +- Support for MongoDB as a backend database. +- Some configuration settings and `bigchaindb` command-line commands were added. In particular, one can specify the database backend (`rethinkdb` or `mongodb`). For MongoDB, one can specify the name of the replicaset. Also for MongoDB, there are new command-line commands to add and remove hosts from the replicaset. See [the Settings & CLI docs](https://docs.bigchaindb.com/projects/server/en/v0.9.0/server-reference/index.html). +- Transaction schema validation. The transaction schema is also used to auto-generate some docs. [Pull Request #880](https://github.com/bigchaindb/bigchaindb/pull/880) +- Vote schema validation. The vote schema is also used to auto-generate some docs. [Pull Request #865](https://github.com/bigchaindb/bigchaindb/pull/865) +- New `ENABLE_WEB_ADMIN` setting in the AWS deployment configuration file. [Pull Request #1015](https://github.com/bigchaindb/bigchaindb/pull/1015) + +### Changed +- The transaction model has changed substantially. @libscott wrote a blog post about the changes and it will be published soon on [the Planetmint Blog](https://blog.bigchaindb.com/). Also, see [the docs about the transaction model](https://docs.bigchaindb.com/projects/server/en/v0.9.0/data-models/transaction-model.html). +- The HTTP API has changed substantially. @diminator wrote a blog post about the changes and it will be published soon on [the Planetmint Blog](https://blog.bigchaindb.com/). Also, see [the docs about the vote model](https://docs.bigchaindb.com/projects/server/en/v0.9.0/data-models/vote-model.html). +- All RethinkDB-specific database calls were replaced with abstract calls to a backend database. +- Some improvements to the Dockerfile, e.g. Pull Requests [#1011](https://github.com/bigchaindb/bigchaindb/pull/1011) and [#1121](https://github.com/bigchaindb/bigchaindb/pull/1121) +- Many improvements to the tests +- We standardized on supporting Ubuntu 16.04 for now (but Planetmint Server also works on similar Linux distros). + +### Removed +- `api_endpoint` was removed from the Planetmint configuration settings. (It was never used anyway.) [Pull Request #821](https://github.com/bigchaindb/bigchaindb/pull/821) +- Removed all remaining StatsD monitoring code, configuration settings, docs, etc. (We'll add another monitoring solution in the future.) [Pull Request #1138](https://github.com/bigchaindb/bigchaindb/pull/1138) + +### Fixed +- Fixed a memory (RAM) overflow problem when under heavy load by bounding the size of the queue at the entrance to the block pipeline. [Pull Request #908](https://github.com/bigchaindb/bigchaindb/pull/908) +- Fixed some logic in block validation. [Pull Request #1130](https://github.com/bigchaindb/bigchaindb/pull/1130) + +### External Contributors +- @amirelemam - [Pull Request #762](https://github.com/bigchaindb/bigchaindb/pull/762) (closed but used as the basis for [Pull Request #1074](https://github.com/bigchaindb/bigchaindb/pull/1074)) +- @utarl - [Pull Request #1019](https://github.com/bigchaindb/bigchaindb/pull/1019) + +### Notes +- There were many additions and changes to the documentation. Fun fact: The JSON in the HTTP API docs is now auto-generated to be consistent with the current code. +- There's a draft spec for a Planetmint Event Stream API and we welcome your feedback. See [Pull Request #1086](https://github.com/bigchaindb/bigchaindb/pull/1086) + + +## [0.8.2] - 2017-01-27 +Tag name: v0.8.2 + +### Fixed +- Fix spend input twice in same transaction + (https://github.com/bigchaindb/bigchaindb/issues/1099). + + +## [0.8.1] - 2017-01-16 +Tag name: v0.8.1 + +### Changed +- Upgrade pysha3 to 1.0.0 (supports official NIST standard). + +### Fixed +- Workaround for rapidjson problem with package metadata extraction + (https://github.com/kenrobbins/python-rapidjson/pull/52). + + +## [0.8.0] - 2016-11-29 +Tag name: v0.8.0 + +### Added +- The big new thing in version 0.8.0 is support for divisible assets, i.e. assets like carrots or thumbtacks, where the initial CREATE transaction can register/create some amount (e.g. 542 carrots), the first TRANSFER transaction can split that amount across multiple owners, and so on. [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) +- Wrote a formal schema for the JSON structure of transactions. [Pull Request #798](https://github.com/bigchaindb/bigchaindb/pull/798) +- New configuration parameter: `backlog_reassign_delay`. [Pull Request #883](https://github.com/bigchaindb/bigchaindb/pull/883) + +### Changed +- CREATE transactions must now be signed by all `owners_before` (rather than by a federation node). [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) +- The user-provided timestamp was removed from the transaction data model (schema). [Pull Request #817](https://github.com/bigchaindb/bigchaindb/pull/817) +- `get_transaction()` will now return a transaction from the backlog, even if there are copies of the transaction in invalid blocks. [Pull Request #793](https://github.com/bigchaindb/bigchaindb/pull/793) +- Several pull requests to introduce a generalized database interface, to move RethinkDB calls into a separate implementation of that interface, and to work on a new MongoDB implementation of that interface. Pull Requests +[#754](https://github.com/bigchaindb/bigchaindb/pull/754), +[#783](https://github.com/bigchaindb/bigchaindb/pull/783), +[#799](https://github.com/bigchaindb/bigchaindb/pull/799), +[#806](https://github.com/bigchaindb/bigchaindb/pull/806), +[#809](https://github.com/bigchaindb/bigchaindb/pull/809), +[#853](https://github.com/bigchaindb/bigchaindb/pull/853) +- Renamed "verifying key" to "public key". Renamed "signing key" to "private key". Renamed "vk" to "pk". [Pull Request #807](https://github.com/bigchaindb/bigchaindb/pull/807) +- `get_transaction_by_asset_id` now ignores invalid transactions. [Pull Request #810](https://github.com/bigchaindb/bigchaindb/pull/810) +- `get_transaction_by_metadata_id` now ignores invalid transactions. [Pull Request #811](https://github.com/bigchaindb/bigchaindb/pull/811) +- Updates to the configs and scripts for deploying a test network on AWS. The example config file deploys virtual machines running Ubuntu 16.04 now. Pull Requests +[#771](https://github.com/bigchaindb/bigchaindb/pull/771), +[#813](https://github.com/bigchaindb/bigchaindb/pull/813) +- Changed logging of transactions on block creation so now it just says the length of the list of transactions, rather than listing all the transactions. [Pull Request #861](https://github.com/bigchaindb/bigchaindb/pull/861) + +### Fixed +- Equality checks with AssetLinks. [Pull Request #825](https://github.com/bigchaindb/bigchaindb/pull/825) +- Bug in `bigchaindb load`. [Pull Request #824](https://github.com/bigchaindb/bigchaindb/pull/824) +- Two issues found with timestamp indexes. [Pull Request #816](https://github.com/bigchaindb/bigchaindb/pull/816) +- Hard-coded `backlog_reassign_delay`. [Pull Request #854](https://github.com/bigchaindb/bigchaindb/pull/854) +- Race condition in `test_stale_monitor.py`. [Pull Request #846](https://github.com/bigchaindb/bigchaindb/pull/846) +- When creating a signed vote, decode the vote signature to a `str`. [Pull Request #869](https://github.com/bigchaindb/bigchaindb/pull/869) +- Bug in AWS deployment scripts. Setting `BIND_HTTP_TO_LOCALHOST` to `False` didn't actually work. It does now. [Pull Request #870](https://github.com/bigchaindb/bigchaindb/pull/870) + +### External Contributors +- @najlachamseddine - [Pull Request #528](https://github.com/bigchaindb/bigchaindb/pull/528) +- @ChristianGaertner - [Pull Request #659](https://github.com/bigchaindb/bigchaindb/pull/659) +- @MinchinWeb - [Pull Request #695](https://github.com/bigchaindb/bigchaindb/pull/695) +- @ckeyer - [Pull Request #785](https://github.com/bigchaindb/bigchaindb/pull/785) + +### Notes +- @ChristianGaertner added a Python style checker (Flake8) to Travis CI, so external contributors should be aware that the Python code in their pull requests will be checked. See [our Python Style Guide](PYTHON_STYLE_GUIDE.md). +- Several additions and changes to the documentation, e.g. Pull Requests +[#690](https://github.com/bigchaindb/bigchaindb/pull/690), +[#764](https://github.com/bigchaindb/bigchaindb/pull/764), +[#766](https://github.com/bigchaindb/bigchaindb/pull/766), +[#769](https://github.com/bigchaindb/bigchaindb/pull/769), +[#777](https://github.com/bigchaindb/bigchaindb/pull/777), +[#800](https://github.com/bigchaindb/bigchaindb/pull/800), +[#801](https://github.com/bigchaindb/bigchaindb/pull/801), +[#802](https://github.com/bigchaindb/bigchaindb/pull/802), +[#803](https://github.com/bigchaindb/bigchaindb/pull/803), +[#819](https://github.com/bigchaindb/bigchaindb/pull/819), +[#827](https://github.com/bigchaindb/bigchaindb/pull/827), +[#859](https://github.com/bigchaindb/bigchaindb/pull/859), +[#872](https://github.com/bigchaindb/bigchaindb/pull/872), +[#882](https://github.com/bigchaindb/bigchaindb/pull/882), +[#883](https://github.com/bigchaindb/bigchaindb/pull/883) + + +## [0.7.0] - 2016-10-28 +Tag name: v0.7.0 += commit: 2dd7f1af27478c529e6d2d916f64daa3fbda3885 +committed: Oct 28, 2016, 4:00 PM GMT+2 + +### Added +- Stale transactions in the `backlog` table now get reassigned to another node (for inclusion in a new block): [Pull Request #359](https://github.com/bigchaindb/bigchaindb/pull/359) +- Many improvements to make the database connection more robust: [Pull Request #623](https://github.com/bigchaindb/bigchaindb/pull/623) +- The new `--dev-allow-temp-keypair` option on `bigchaindb start` will generate a temporary keypair if no keypair is found. The `Dockerfile` was updated to use this. [Pull Request #635](https://github.com/bigchaindb/bigchaindb/pull/635) +- The AWS deployment scripts now allow you to: + - specify the AWS security group as a configuration parameter: [Pull Request #620](https://github.com/bigchaindb/bigchaindb/pull/620) + - tell RethinkDB to bind HTTP to localhost (a more secure setup; now the default in the example config file): [Pull Request #666](https://github.com/bigchaindb/bigchaindb/pull/666) + +### Changed +- Integrated the new `Transaction` model. This was a **big** change; 49 files changed. [Pull Request #641](https://github.com/bigchaindb/bigchaindb/pull/641) +- Merged "common" code (used by Planetmint Server and the Python driver), which used to be in its own repository (`bigchaindb/bigchaindb-common`), into the main `bigchaindb/bigchaindb` repository (this one): [Pull Request #742](https://github.com/bigchaindb/bigchaindb/pull/742) +- Integrated the new digital asset model. This changed the data structure of a transaction and will make it easier to support divisible assets in the future. [Pull Request #680](https://github.com/bigchaindb/bigchaindb/pull/680) +- Transactions are now deleted from the `backlog` table _after_ a block containing them is written to the `bigchain` table: [Pull Request #609](https://github.com/bigchaindb/bigchaindb/pull/609) +- Changed the example AWS deployment configuration file: [Pull Request #665](https://github.com/bigchaindb/bigchaindb/pull/665) +- Support for version 0.5.0 of the `cryptoconditions` Python package. Note that this means you must now install `ffi.h` (e.g. `sudo apt-get install libffi-dev` on Ubuntu). See Pull Requests [#685](https://github.com/bigchaindb/bigchaindb/pull/685) and [#698](https://github.com/bigchaindb/bigchaindb/pull/698) +- Updated some database access code: Pull Requests [#676](https://github.com/bigchaindb/bigchaindb/pull/676) and [#701](https://github.com/bigchaindb/bigchaindb/pull/701) + +### Fixed +- Internally, when a transaction is in the `backlog` table, it carries some extra book-keeping fields: + 1. an `assignment_timestamp` (i.e. the time when it was assigned to a node), which is used to determine if it has gone stale. + 2. an `assignee`: the public key of the node it was assigned to. +- The `assignment_timestamp` wasn't removed before writing the transaction to a block. That was fixed in [Pull Request #627](https://github.com/bigchaindb/bigchaindb/pull/627) +- The `assignment_timestamp` and `assignee` weren't removed in the response to an HTTP API request sent to the `/api/v1/transactions/` endpoint. That was fixed in [Pull Request #646](https://github.com/bigchaindb/bigchaindb/pull/646) +- When validating a TRANSFER transaction, if any fulfillment refers to a transaction that's _not_ in a valid block, then the transaction isn't valid. This wasn't checked before but it is now. [Pull Request #629](https://github.com/bigchaindb/bigchaindb/pull/629) + +### External Contributors +- @MinchinWeb - [Pull Request #696](https://github.com/bigchaindb/bigchaindb/pull/696) + +### Notes +- We made a small change to how we do version labeling. Going forward, we will have the version label set to 0.X.Y.dev in the master branch as we work on what will eventually be released as version 0.X.Y. The version number will only be changed to 0.X.Y just before the release. This version labeling scheme began with [Pull Request #752](https://github.com/bigchaindb/bigchaindb/pull/752) +- Several additions and changes to the documentation, e.g. Pull Requests +[#618](https://github.com/bigchaindb/bigchaindb/pull/618), +[#621](https://github.com/bigchaindb/bigchaindb/pull/621), +[#625](https://github.com/bigchaindb/bigchaindb/pull/625), +[#645](https://github.com/bigchaindb/bigchaindb/pull/645), +[#647](https://github.com/bigchaindb/bigchaindb/pull/647), +[#648](https://github.com/bigchaindb/bigchaindb/pull/648), +[#650](https://github.com/bigchaindb/bigchaindb/pull/650), +[#651](https://github.com/bigchaindb/bigchaindb/pull/651), +[#653](https://github.com/bigchaindb/bigchaindb/pull/653), +[#655](https://github.com/bigchaindb/bigchaindb/pull/655), +[#656](https://github.com/bigchaindb/bigchaindb/pull/656), +[#657](https://github.com/bigchaindb/bigchaindb/pull/657), +[#667](https://github.com/bigchaindb/bigchaindb/pull/667), +[#668](https://github.com/bigchaindb/bigchaindb/pull/668), +[#669](https://github.com/bigchaindb/bigchaindb/pull/669), +[#673](https://github.com/bigchaindb/bigchaindb/pull/673), +[#678](https://github.com/bigchaindb/bigchaindb/pull/678), +[#684](https://github.com/bigchaindb/bigchaindb/pull/684), +[#688](https://github.com/bigchaindb/bigchaindb/pull/688), +[#699](https://github.com/bigchaindb/bigchaindb/pull/699), +[#705](https://github.com/bigchaindb/bigchaindb/pull/705), +[#737](https://github.com/bigchaindb/bigchaindb/pull/737), +[#748](https://github.com/bigchaindb/bigchaindb/pull/748), +[#753](https://github.com/bigchaindb/bigchaindb/pull/753), +[#757](https://github.com/bigchaindb/bigchaindb/pull/757), +[#759](https://github.com/bigchaindb/bigchaindb/pull/759), and more + + +## [0.6.0] - 2016-09-01 +Tag name: v0.6.0 += commit: bfc86e0295c7d1ef0acd3c275c125798bd5b0dfd +committed: Sep 1, 2016, 2:15 PM GMT+2 + +### Added +- Support for multiple operations in the ChangeFeed class: [Pull Request #569](https://github.com/bigchaindb/bigchaindb/pull/569) +- Instructions, templates and code for deploying a starter node on AWS using Terraform and Ansible: Pull Requests +[#572](https://github.com/bigchaindb/bigchaindb/pull/572), +[#589](https://github.com/bigchaindb/bigchaindb/pull/589), +[#600](https://github.com/bigchaindb/bigchaindb/pull/600), +[#605](https://github.com/bigchaindb/bigchaindb/pull/605), +[#610](https://github.com/bigchaindb/bigchaindb/pull/610) +- Check that the majority of votes on a block agree on the previous block. If they don't, consider the block invalid. [Pull Request #565](https://github.com/bigchaindb/bigchaindb/pull/565) + +### Changed +- Set RethinkDB `read-mode='majority'` everywhere: [Pull Request #497](https://github.com/bigchaindb/bigchaindb/pull/497) +- Ported election logic and voting logic to the new pipeline architecture: Pull Requests [#510](https://github.com/bigchaindb/bigchaindb/pull/510) and [#515](https://github.com/bigchaindb/bigchaindb/pull/515) +- Moved the transaction (model) `version` inside the `transaction` (in the transaction data model): [Pull Request #518](https://github.com/bigchaindb/bigchaindb/pull/518) +- Changed how the Planetmint config file (JSON) gets written so it's easier for humans to read: [Pull Request #522](https://github.com/bigchaindb/bigchaindb/pull/522) +- Improved and expanded the GET/POST endpoints for transactions (in the HTTP API): [Pull Request #563](https://github.com/bigchaindb/bigchaindb/pull/563) +- Changed the AWS cluster deployment scripts so that the deployer now generates their keypair locally, rather than having Amazon do it: [Pull Request #567](https://github.com/bigchaindb/bigchaindb/pull/567) +- When a transaction is retrieved by `get_transaction`, a `validity` field is added with a value one of `valid`, `undecided`, or `backlog`: [Pull Request #574](https://github.com/bigchaindb/bigchaindb/pull/574) +- Renamed `current_owners` and `new_owners` (in the data models) to `owners_before` and `owners_after`, respectively (i.e. before/after *the transaction*): [Pull Request #578](https://github.com/bigchaindb/bigchaindb/pull/578) +- Use `flask_restful` and class-based views for realizing the HTTP API: [Pull Request #588](https://github.com/bigchaindb/bigchaindb/pull/588) + +### Fixed +- Fixed the confusing error message when there was a syntax error in the Planetmint config file: [Pull Request #531](https://github.com/bigchaindb/bigchaindb/pull/531) +- Fixed `write_transaction` so it no longer has the side effect of adding `assignee` to a transaction that is being processed: [Pull Request #606](https://github.com/bigchaindb/bigchaindb/pull/606) + +### External Contributors +- @eladve - [Pull Request #518](https://github.com/bigchaindb/bigchaindb/pull/518) +- @d01phin - Pull Requests [#522](https://github.com/bigchaindb/bigchaindb/pull/522) and [#531](https://github.com/bigchaindb/bigchaindb/pull/531) +- @Kentoseth - [Pull Request #537](https://github.com/bigchaindb/bigchaindb/pull/537) + +### Notes +- Several additions and changes to the documentation, e.g. Pull Requests +[#523](https://github.com/bigchaindb/bigchaindb/pull/523), +[#532](https://github.com/bigchaindb/bigchaindb/pull/532), +[#537](https://github.com/bigchaindb/bigchaindb/pull/537), +[#539](https://github.com/bigchaindb/bigchaindb/pull/539), +[#610](https://github.com/bigchaindb/bigchaindb/pull/610), and more + + +## [0.5.1] - 2016-07-29 +Tag name: v0.5.1 += commit: ff042b5954abe48c7264d43128d52584eab2a806 +committed: Jul 29, 2016, 2:38 PM GMT+2 + +### Added +- New third table, the 'votes' table: [Pull Request #379](https://github.com/bigchaindb/bigchaindb/pull/379) +- Added `get_tx_by_payload_uuid()` including index: [Pull Request #411](https://github.com/bigchaindb/bigchaindb/pull/411) +- Ability to deploy a test cluster on AWS using Amazon Elastic Block Store (EBS) for storage: [Pull Request #469](https://github.com/bigchaindb/bigchaindb/pull/469) +- Ability to add different-size payloads to transactions when filling the backlog for benchmarking: [Pull Request #273](https://github.com/bigchaindb/bigchaindb/pull/273) + +### Changed +- Votes are no longer appended to the blocks inside the 'bigchain' table. They're now written to their own table, the 'votes' table: [Pull Request #379](https://github.com/bigchaindb/bigchaindb/pull/379) +- Refactored how blocks get constructed using the new approach to doing multiprocessing, using the `multipipes` package: [Pull Request #484](https://github.com/bigchaindb/bigchaindb/pull/484) +- Changed all queries to use `read_mode='majority'`: [Pull Request #497](https://github.com/bigchaindb/bigchaindb/pull/497) +- Revised how base software gets deployed on AWS instances: [Pull Request #478](https://github.com/bigchaindb/bigchaindb/pull/478) +- Refactored `db.utils.init()`: [Pull Request #471](https://github.com/bigchaindb/bigchaindb/pull/471) + +### External Contributors +- @shauns - [Pull Request #411](https://github.com/bigchaindb/bigchaindb/pull/411) +- @lonelypeanut - [Pull Request #479](https://github.com/bigchaindb/bigchaindb/pull/479) +- @lluminita - Pull Requests [#435](https://github.com/bigchaindb/bigchaindb/pull/435) & [#471](https://github.com/bigchaindb/bigchaindb/pull/471) + +### Notes +- Several additions and changes to the documentation: Pull Requests +[#416](https://github.com/bigchaindb/bigchaindb/pull/416), +[#417](https://github.com/bigchaindb/bigchaindb/pull/417), +[#418](https://github.com/bigchaindb/bigchaindb/pull/418), +[#420](https://github.com/bigchaindb/bigchaindb/pull/420), +[#421](https://github.com/bigchaindb/bigchaindb/pull/421), +[#422](https://github.com/bigchaindb/bigchaindb/pull/422), +[#423](https://github.com/bigchaindb/bigchaindb/pull/423), +[#425](https://github.com/bigchaindb/bigchaindb/pull/425), +[#428](https://github.com/bigchaindb/bigchaindb/pull/428), +[#430](https://github.com/bigchaindb/bigchaindb/pull/430), +[#431](https://github.com/bigchaindb/bigchaindb/pull/431), +[#435](https://github.com/bigchaindb/bigchaindb/pull/435), +[#442](https://github.com/bigchaindb/bigchaindb/pull/442), +[#472](https://github.com/bigchaindb/bigchaindb/pull/472), +[#481](https://github.com/bigchaindb/bigchaindb/pull/481) + + +## [0.5.0] - 2016-07-04 +Tag name: v0.5.0 += commit: 38329531304952128b48f2e5603db5fa08069c26 +committed: July 4, 2016, 1:07 PM GMT+2 + +### Added +- New `bigchaindb set-replicas` subcommand: [Pull Request #392](https://github.com/bigchaindb/bigchaindb/pull/392) +- Informative JSON message when one makes a request to the root endpoint of the HTTP client-server API: [Pull Request #367](https://github.com/bigchaindb/bigchaindb/pull/367) +- Return HTTP response code 404 when a transaction is not found: [Pull Request #369](https://github.com/bigchaindb/bigchaindb/pull/369) + +### Changed +- Changed the order in which configuration settings get their values. If a setting is set by an environment variable, then that value will be _the_ value, regardless of whether another value is set in a local config file. Also added a method to programattically update the config settings. [Pull Request #395](https://github.com/bigchaindb/bigchaindb/pull/395) +- Changed the definition of `util.sign_tx()`. It now has a third, optional argument: a Bigchain instance. [Pull Request #410](https://github.com/bigchaindb/bigchaindb/pull/410) + +### Notes +- Several additions and changes to the documentation: Pull Requests +[#388](https://github.com/bigchaindb/bigchaindb/pull/388), +[#393](https://github.com/bigchaindb/bigchaindb/pull/393), +[#397](https://github.com/bigchaindb/bigchaindb/pull/397), +[#402](https://github.com/bigchaindb/bigchaindb/pull/402), +[#403](https://github.com/bigchaindb/bigchaindb/pull/403), +[#406](https://github.com/bigchaindb/bigchaindb/pull/406), +[#408](https://github.com/bigchaindb/bigchaindb/pull/408) + + +## [0.4.2] - 2016-06-15 +Tag name: v0.4.2 += commit: 7ce6c3980cf70437d7ce716a67f069afa8ecb79e +committed: June 15, 2016, 1:42 PM GMT+2 + +### Added +- Report the Planetmint version number when starting Planetmint: [Pull Request #385](https://github.com/bigchaindb/bigchaindb/pull/385) + +### Changed +- Round timestamps to a precision of one second, and replace payload hash with payload UUID in transactions: [Pull Request #384](https://github.com/bigchaindb/bigchaindb/pull/384) +- Updated cryptoconditions API usage: [Pull Request #373](https://github.com/bigchaindb/bigchaindb/pull/373) + + +## [0.4.1] - 2016-06-13 +Tag name: v0.4.1 += commit: 9c4aa987bcbc294b6a5c3069e6c45a7ed77a4068 +committed: June 13, 2016, 9:52 AM GMT+2 + +### Added +- Revert `bigchain` deletes: [Pull Request #330](https://github.com/bigchaindb/bigchaindb/pull/330) + +### Changed +- Use inverted threshold condition instead of negative weights for escrow: [Pull Request #355](https://github.com/bigchaindb/bigchaindb/pull/355) + +### Fixed +- Removed duplicate `pytest` in `setup.py`: [Pull Request #365](https://github.com/bigchaindb/bigchaindb/pull/365) + +### Notes +- There were several additions and changes to the documentation: Pull Requests +[#343](https://github.com/bigchaindb/bigchaindb/pull/343), +[#363](https://github.com/bigchaindb/bigchaindb/pull/363), +[#364](https://github.com/bigchaindb/bigchaindb/pull/364), +[#366](https://github.com/bigchaindb/bigchaindb/pull/366), +[#370](https://github.com/bigchaindb/bigchaindb/pull/370), +[#372](https://github.com/bigchaindb/bigchaindb/pull/372) + + +## [0.4.0] - 2016-05-27 +Tag name: v0.4.0 += commit: a89399c4f9fcdf82df73e0d8191af9e539d8d081 +committed: May 27, 2016, 1:42 PM GMT+2 + +### Added +- Support for escrow (possible because of two other new things: cryptoconditions with inverters, and a timeout condition): [Pull Request #329](https://github.com/bigchaindb/bigchaindb/pull/329) +- Caching of calls to `load_consensus_plugin()`, using [`@lru_cache`](https://docs.python.org/3/library/functools.html#functools.lru_cache). This speeds up the instantiation of `Bigchain` objects and greatly improves overall performance. [Pull Request #271](https://github.com/bigchaindb/bigchaindb/pull/271) +- New `Dockerfile-dev` Docker file to make it easier for developers to _develop_ Planetmint with Docker. One can run all unit tests with Docker again. [Pull Request #313](https://github.com/bigchaindb/bigchaindb/pull/313) +- Transactions in invalid blocks are copied to the backlog: [Pull Request #221](https://github.com/bigchaindb/bigchaindb/pull/221). +- Queries to the bigchain table now ignore invalid blocks: [Pull Request #324](https://github.com/bigchaindb/bigchaindb/issues/324) +- Use secondary index on get_transaction: [Pull Request #324](https://github.com/bigchaindb/bigchaindb/issues/324) +- New `bigchaindb` command to set the number of RethinkDB shards (in both tables): [Pull Request #258](https://github.com/bigchaindb/bigchaindb/pull/258) +- Better handling of an outdated `setuptools`: [Pull Request #279](https://github.com/bigchaindb/bigchaindb/pull/279) + +### Changed +- The block processes now use GroupProcess: [Pull Request #267](https://github.com/bigchaindb/bigchaindb/pull/267) +- Replaced the `json` Python package with `rapidjson` (a Python wrapper for a fast JSON parser/generator written in C++), to speed up JSON serialization and deserialization: [Pull Request #318](https://github.com/bigchaindb/bigchaindb/pull/318) +- Overhauled `ROADMAP.md` and moved it to [the bigchaindb/org repository](https://github.com/bigchaindb/org): Pull Requests +[#282](https://github.com/bigchaindb/bigchaindb/pull/282), +[#306](https://github.com/bigchaindb/bigchaindb/pull/306), +[#308](https://github.com/bigchaindb/bigchaindb/pull/308), +[#325](https://github.com/bigchaindb/bigchaindb/pull/325) +- AWS deployment has better support for [New Relic Server Monitoring](https://newrelic.com/server-monitoring): [Pull Request #316](https://github.com/bigchaindb/bigchaindb/pull/316) +- AWS deployment script now reads from a configuration file: [Pull Request #291](https://github.com/bigchaindb/bigchaindb/pull/291) +- AWS deployment script doesn't auto-start the Planetmint servers anymore: [Pull Request #257](https://github.com/bigchaindb/bigchaindb/pull/257) + +### Fixed +- Bug related to transaction malleability: [Pull Request #281](https://github.com/bigchaindb/bigchaindb/pull/281) + +### Notes +You can now see a big-picture view of all Planetmint repositories on [a waffle.io board](https://waffle.io/bigchaindb/org). + + +## [0.3.0] - 2016-05-03 +Tag name: v0.3.0 += commit: a97c54e82be954a1411e5bfe0f09a9c631309f1e +committed: May 3, 2016, 11:52 AM GMT+2 + +### Added +- Crypto-conditions specs according to the Interledger protocol: [Pull Request #174](https://github.com/bigchaindb/bigchaindb/pull/174) +- Added support for anonymous hashlocked conditions and fulfillments: [Pull Request #211](https://github.com/bigchaindb/bigchaindb/pull/211) + +### Changed +- Several improvements to the AWS deployment scripts: [Pull Request #227](https://github.com/bigchaindb/bigchaindb/pull/227) + +### Fixed +- Bug related to block validation: [Pull Request #233](https://github.com/bigchaindb/bigchaindb/pull/233) + +### Notes +This release completely refactored the structure of the transactions and broke compatibility with older versions +of Planetmint. The refactor of the transactions was made in order to add support for multiple inputs/outputs and +the crypto-conditions specs from the Interledger protocol. + +We also updated the RethinkDB Python drivers so you need to upgrade to RethinkDB v2.3+ + + +## [0.2.0] - 2016-04-26 +Tag name: v0.2.0 += commit: 0c4a2b380aabdcf50fa2d7fb351c290aaedc3db7 +committed: April 26, 2016, 11:09 AM GMT+2 + +### Added +- Ability to use environment variables to set (or partially set) configuration settings: [Pull Request #153](https://github.com/bigchaindb/bigchaindb/pull/153) +- `bigchaindb --export-my-pubkey`: [Pull Request #186](https://github.com/bigchaindb/bigchaindb/pull/186) +- `bigchaindb --version`, and one central source for the current version (`version.py`): [Pull Request #208](https://github.com/bigchaindb/bigchaindb/pull/208) +- AWS deployment scripts: Pull Requests +[#160](https://github.com/bigchaindb/bigchaindb/pull/160), +[#166](https://github.com/bigchaindb/bigchaindb/pull/166), +[#172](https://github.com/bigchaindb/bigchaindb/pull/172), +[#203](https://github.com/bigchaindb/bigchaindb/pull/203) +- `codecov.yml`: [Pull Request #161](https://github.com/bigchaindb/bigchaindb/pull/161) +- `CHANGELOG.md` (this file): [Pull Request #117](https://github.com/bigchaindb/bigchaindb/pull/117) +- Signatures using Ed25519: Pull Requests +[#138](https://github.com/bigchaindb/bigchaindb/pull/138), +[#152](https://github.com/bigchaindb/bigchaindb/pull/152) +- Multisig support: [Pull Request #107](https://github.com/bigchaindb/bigchaindb/pull/107) +- HTTP Server & Web API: Pull Requests +[#102](https://github.com/bigchaindb/bigchaindb/pull/102), +[#150](https://github.com/bigchaindb/bigchaindb/pull/150), +[#155](https://github.com/bigchaindb/bigchaindb/pull/155), +[#183](https://github.com/bigchaindb/bigchaindb/pull/183) +- Python driver/SDK/API: [Pull Request #102](https://github.com/bigchaindb/bigchaindb/pull/102) +- Python Style Guide: [Pull Request #89](https://github.com/bigchaindb/bigchaindb/pull/89) +- Monitoring & dashboard tools: Pull Requests +[#72](https://github.com/bigchaindb/bigchaindb/pull/72), +[#181](https://github.com/bigchaindb/bigchaindb/pull/181) + +### Changed +- Rewrote [`README.md`](README.md) into four sets of links: Pull Requests [#80](https://github.com/bigchaindb/bigchaindb/pull/80) and [#115](https://github.com/bigchaindb/bigchaindb/pull/115) + +### Fixed +- Bug related to config overwrite: [Pull Request #97](https://github.com/bigchaindb/bigchaindb/pull/97) +- Bug related to running the `bigchaindb-benchmark load` on docker [Pull Request #225](https://github.com/bigchaindb/bigchaindb/pull/225) + +## External Contributors +- [@thedoctor](https://github.com/thedoctor): Pull Requests +[#99](https://github.com/bigchaindb/bigchaindb/pull/99), +[#136](https://github.com/bigchaindb/bigchaindb/pull/136) +- [@roderik](https://github.com/roderik): [Pull Request #162](https://github.com/bigchaindb/bigchaindb/pull/162) + + +## [0.1.5] - 2016-04-20 +Tag name: v0.1.5 += commit: 9f62cddbaf44167692cfee71db707bce93e3395f +committed: April 20, 2016, 3:31 PM GMT+2 + +### Fixed +- [Issue #71](https://github.com/bigchaindb/bigchaindb/issues/71) (Voter is not validating blocks correctly when checking for double spends) in [Pull Request #76](https://github.com/bigchaindb/bigchaindb/pull/76) + + +## [0.1.4] - 2016-02-22 +Tag name: v0.1.4 += commit: c4c850f480bc9ae72df2a54f81c0825b6fb4ed62 +committed: Feb 22, 2016, 11:51 AM GMT+1 + +### Added +- Added to `classifiers` to setup.py + +### Changed +- Allow running pytest tests in parallel (using [xdist](http://pytest.org/latest/xdist.html)): [Pull Request #65](https://github.com/bigchaindb/bigchaindb/pull/65) +- Allow non-interactive first start: [Pull Request #64](https://github.com/bigchaindb/bigchaindb/pull/64) to resolve [Issue #58](https://github.com/bigchaindb/bigchaindb/issues/58) + + +## [0.1.3] - 2016-02-16 +Tag name: v0.1.3 += commit 8926e3216c1ee39b9bc332e5ef1df2a8901262dd +committed Feb 16, 2016, 11:37 AM GMT+1 + +### Changed +- Changed from using Git Flow to GitHub flow (but with `develop` as the default branch). + + +## [0.1.2] - 2016-02-15 +Tag name: v0.1.2 += commit d2ff24166d69dda68dd7b4a24a88279b1d37e222 +committed Feb 15, 2016, 2:23 PM GMT+1 + +### Added +- Various tests + +### Fixed +- Fix exception when running `start`: [Pull Request #32](https://github.com/bigchaindb/bigchaindb/pull/32) resolved [Issue #35] + +## [0.1.1] - 2016-02-15 +Tag name: v0.1.1 += commit 2a025448b29fe7056760de1039c73bbcfe992461 +committed Feb 15, 2016, 10:48 AM GMT+1 + +### Added +- "release 0.1.1": [Pull Request #37](https://github.com/bigchaindb/bigchaindb/pull/37) + +### Removed +- `tox.ini` [Pull Request #18](https://github.com/bigchaindb/bigchaindb/pull/18) +- `requirements.txt` in the root directory, and the entire `requirements/` directory: [Pull Request #14](https://github.com/bigchaindb/bigchaindb/pull/14) + +### Fixed +- Hotfix for AttributeError, fixed [Issue #27](https://github.com/bigchaindb/bigchaindb/issues/27) + + +## [0.1.0] - 2016-02-10 +Tag name: v0.1.0 += commit 8539e8dc2d036a4e0a866a3fb9e55889503254d5 +committed Feb 10, 2016, 10:04 PM GMT+1 + +The first public release of Planetmint, including: + +- Initial Planetmint Server code, including many tests and some code for benchmarking. +- Initial documentation (in `bigchaindb/docs`). +- Initial `README.md`, `ROADMAP.md`, `CODE_OF_CONDUCT.md`, and `CONTRIBUTING.md`. +- Packaging for PyPI, including `setup.py` and `setup.cfg`. +- Initial `Dockerfile` and `docker-compose.yml` (for deployment using Docker and Docker Compose). +- Initial `.gitignore` (list of things for git to ignore). +- Initial `.travis.yml` (used by Travis CI). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..bca9eab --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,57 @@ + + +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute to the project. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, nationality, or species--no picking on Wrigley for being a buffalo! + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Deliberate intimidation +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior directed at yourself or another community member may be +reported by contacting a project maintainer at [contact@bigchaindb.com](mailto:contact@bigchaindb.com). All +complaints will be reviewed and investigated and will result in a response that +is appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..677aa1c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.6 +LABEL maintainer "contact@ipdb.global" +RUN mkdir -p /usr/src/app +COPY . /usr/src/app/ +WORKDIR /usr/src/app +RUN apt-get -qq update \ + && apt-get -y upgrade \ + && apt-get install -y jq \ + && pip install . \ + && apt-get autoremove \ + && apt-get clean + +VOLUME ["/data", "/certs"] + +ENV PYTHONUNBUFFERED 0 +ENV PLANETMINT_CONFIG_PATH /data/.bigchaindb +ENV PLANETMINT_SERVER_BIND 0.0.0.0:9984 +ENV PLANETMINT_WSSERVER_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_SCHEME ws +ENV PLANETMINT_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_ADVERTISED_SCHEME ws +ENV PLANETMINT_WSSERVER_ADVERTISED_PORT 9985 +ENTRYPOINT ["bigchaindb"] +CMD ["start"] diff --git a/Dockerfile-all-in-one b/Dockerfile-all-in-one new file mode 100644 index 0000000..11d8d20 --- /dev/null +++ b/Dockerfile-all-in-one @@ -0,0 +1,51 @@ +FROM alpine:3.9 +LABEL maintainer "contact@ipdb.global" + +ARG TM_VERSION=v0.31.5 +RUN mkdir -p /usr/src/app +ENV HOME /root +COPY . /usr/src/app/ +WORKDIR /usr/src/app + +RUN apk --update add sudo bash \ + && apk --update add python3 openssl ca-certificates git \ + && apk --update add --virtual build-dependencies python3-dev \ + libffi-dev openssl-dev build-base jq \ + && apk add --no-cache libstdc++ dpkg gnupg \ + && pip3 install --upgrade pip cffi \ + && pip install -e . \ + && apk del build-dependencies \ + && rm -f /var/cache/apk/* + +# Install mongodb and monit +RUN apk --update add mongodb monit + +# Install Tendermint +RUN wget https://github.com/tendermint/tendermint/releases/download/${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip \ + && unzip tendermint_${TM_VERSION}_linux_amd64.zip \ + && mv tendermint /usr/local/bin/ \ + && rm tendermint_${TM_VERSION}_linux_amd64.zip + +ENV TMHOME=/tendermint + +# Set permissions required for mongodb +RUN mkdir -p /data/db /data/configdb \ + && chown -R mongodb:mongodb /data/db /data/configdb + +# Planetmint enviroment variables +ENV PLANETMINT_DATABASE_PORT 27017 +ENV PLANETMINT_DATABASE_BACKEND localmongodb +ENV PLANETMINT_SERVER_BIND 0.0.0.0:9984 +ENV PLANETMINT_WSSERVER_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_SCHEME ws + +ENV PLANETMINT_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_ADVERTISED_SCHEME ws +ENV PLANETMINT_TENDERMINT_PORT 26657 + +VOLUME /data/db /data/configdb /tendermint + +EXPOSE 27017 28017 9984 9985 26656 26657 26658 + +WORKDIR $HOME +ENTRYPOINT ["/usr/src/app/pkg/scripts/all-in-one.bash"] diff --git a/Dockerfile-alpine b/Dockerfile-alpine new file mode 100644 index 0000000..6663deb --- /dev/null +++ b/Dockerfile-alpine @@ -0,0 +1,30 @@ +FROM alpine:latest +LABEL maintainer "contact@ipdb.global" +RUN mkdir -p /usr/src/app +COPY . /usr/src/app/ +WORKDIR /usr/src/app +RUN apk --update add sudo \ + && apk --update add python3 py-pip openssl ca-certificates git\ + && apk --update add --virtual build-dependencies python3-dev \ + libffi-dev openssl-dev build-base \ + && apk add --no-cache libstdc++ \ + && pip3 install --upgrade pip cffi \ + && pip install -e . \ + && apk del build-dependencies \ + && rm -f /var/cache/apk/* +# When developing with Python in a docker container, we are using PYTHONBUFFERED +# to force stdin, stdout and stderr to be totally unbuffered and to capture logs/outputs +ENV PYTHONUNBUFFERED 0 + +ENV PLANETMINT_DATABASE_PORT 27017 +ENV PLANETMINT_DATABASE_BACKEND $backend +ENV PLANETMINT_SERVER_BIND 0.0.0.0:9984 +ENV PLANETMINT_WSSERVER_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_SCHEME ws + +ENV PLANETMINT_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_ADVERTISED_SCHEME ws + +ENV PLANETMINT_TENDERMINT_PORT 26657 +ARG backend +RUN bigchaindb -y configure "$backend" \ No newline at end of file diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 0000000..fbe11e6 --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,35 @@ +ARG python_version=3.6 +FROM python:${python_version} +LABEL maintainer "contact@ipdb.global" + +RUN apt-get update \ + && apt-get install -y git \ + && pip install -U pip \ + && apt-get autoremove \ + && apt-get clean + +ARG backend +ARG abci_status + +# When developing with Python in a docker container, we are using PYTHONBUFFERED +# to force stdin, stdout and stderr to be totally unbuffered and to capture logs/outputs +ENV PYTHONUNBUFFERED 0 + +ENV PLANETMINT_DATABASE_PORT 27017 +ENV PLANETMINT_DATABASE_BACKEND $backend +ENV PLANETMINT_SERVER_BIND 0.0.0.0:9984 +ENV PLANETMINT_WSSERVER_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_SCHEME ws + +ENV PLANETMINT_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV PLANETMINT_WSSERVER_ADVERTISED_SCHEME ws + +ENV PLANETMINT_TENDERMINT_PORT 26657 + +ENV PLANETMINT_CI_ABCI ${abci_status} + +RUN mkdir -p /usr/src/app +COPY . /usr/src/app/ +WORKDIR /usr/src/app +RUN pip install -e .[dev] +RUN bigchaindb -y configure diff --git a/LICENSE b/LICENSE index 0ad25db..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,661 +1,201 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES.md b/LICENSES.md new file mode 100644 index 0000000..5dc2736 --- /dev/null +++ b/LICENSES.md @@ -0,0 +1,24 @@ +# Copyrights and Licenses + +## Copyrights + +For all the code and documentation in this repository, the copyright is owned by one or more of the following: + +- Planetmint GmbH +- A Planetmint contributor who agreed to a Planetmint Contributor License Agreement (CLA) with Planetmint GmbH. (See [BEP-16](https://github.com/bigchaindb/BEPs/tree/master/16).) +- A Planetmint contributor who signed off on the Developer Certificate of Origin (DCO) for all their contributions. (See [BEP-24](https://github.com/bigchaindb/BEPs/tree/master/24).) +- (Rarely, see the **Exceptions Section** below) A third pary who licensed the code in question under an open source license. + +## Code Licenses + +All code in this repository, including short code snippets in the documentation, but not including the **Exceptions** noted below, is licensed under the Apache License, Version 2.0, the full text of which can be found at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). + +For the licenses on all other Planetmint-related code (i.e. in other repositories), see the LICENSE file in the associated repository. + +## Documentation Licenses + +The official Planetmint documentation, _except for the short code snippets embedded within it_, is licensed under a Creative Commons Attribution 4.0 International license, the full text of which can be found at [http://creativecommons.org/licenses/by/4.0/legalcode](http://creativecommons.org/licenses/by/4.0/legalcode). + +## Exceptions + +The contents of the `k8s/nginx-openresty/` directory are licensed as described in the `LICENSE.md` file in that directory. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0195f87 --- /dev/null +++ b/Makefile @@ -0,0 +1,144 @@ +.PHONY: help run start stop logs test test-unit test-unit-watch test-acceptance cov doc doc-acceptance clean reset release dist check-deps clean-build clean-pyc clean-test + +.DEFAULT_GOAL := help + + +############################# +# Open a URL in the browser # +############################# +define BROWSER_PYSCRIPT +import os, webbrowser, sys +try: + from urllib import pathname2url +except: + from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + + +################################## +# Display help for this makefile # +################################## +define PRINT_HELP_PYSCRIPT +import re, sys + +print("Planetmint 2.0 developer toolbox") +print("--------------------------------") +print("Usage: make COMMAND") +print("") +print("Commands:") +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print(" %-16s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +################## +# Basic commands # +################## +DOCKER := docker +DC := docker-compose +BROWSER := python -c "$$BROWSER_PYSCRIPT" +HELP := python -c "$$PRINT_HELP_PYSCRIPT" +ECHO := /usr/bin/env echo + +IS_DOCKER_COMPOSE_INSTALLED := $(shell command -v docker-compose 2> /dev/null) + +################ +# Main targets # +################ + +help: ## Show this help + @$(HELP) < $(MAKEFILE_LIST) + +run: check-deps ## Run Planetmint from source (stop it with ctrl+c) + # although bigchaindb has tendermint and mongodb in depends_on, + # launch them first otherwise tendermint will get stuck upon sending yet another log + # due to some docker-compose issue; does not happen when containers are run as daemons + @$(DC) up --no-deps mongodb tendermint bigchaindb + +start: check-deps ## Run Planetmint from source and daemonize it (stop with `make stop`) + @$(DC) up -d bigchaindb + +stop: check-deps ## Stop Planetmint + @$(DC) stop + +logs: check-deps ## Attach to the logs + @$(DC) logs -f bigchaindb + +test: check-deps test-unit test-acceptance ## Run unit and acceptance tests + +test-unit: check-deps ## Run all tests once + @$(DC) up -d bdb + @$(DC) exec bigchaindb pytest + +test-unit-watch: check-deps ## Run all tests and wait. Every time you change code, tests will be run again + @$(DC) run --rm --no-deps bigchaindb pytest -f + +test-acceptance: check-deps ## Run all acceptance tests + @./run-acceptance-test.sh + +cov: check-deps ## Check code coverage and open the result in the browser + @$(DC) run --rm bigchaindb pytest -v --cov=bigchaindb --cov-report html + $(BROWSER) htmlcov/index.html + +doc: check-deps ## Generate HTML documentation and open it in the browser + @$(DC) run --rm --no-deps bdocs make -C docs/root html + @$(DC) run --rm --no-deps bdocs make -C docs/server html + @$(DC) run --rm --no-deps bdocs make -C docs/contributing html + $(BROWSER) docs/root/build/html/index.html + +doc-acceptance: check-deps ## Create documentation for acceptance tests + @$(DC) run --rm python-acceptance pycco -i -s /src -d /docs + $(BROWSER) acceptance/python/docs/index.html + +clean: clean-build clean-pyc clean-test ## Remove all build, test, coverage and Python artifacts + @$(ECHO) "Cleaning was successful." + +reset: check-deps ## Stop and REMOVE all containers. WARNING: you will LOSE all data stored in Planetmint. + @$(DC) down + +release: dist ## package and upload a release + twine upload dist/* + +dist: clean ## builds source (and not for now, wheel package) + python setup.py sdist + # python setup.py bdist_wheel + ls -l dist + +############### +# Sub targets # +############### + +check-deps: +ifndef IS_DOCKER_COMPOSE_INSTALLED + @$(ECHO) "Error: docker-compose is not installed" + @$(ECHO) + @$(ECHO) "You need docker-compose to run this command. Check out the official docs on how to install it in your system:" + @$(ECHO) "- https://docs.docker.com/compose/install/" + @$(ECHO) + @$(DC) # docker-compose is not installed, so we call it to generate an error and exit +endif + +clean-build: # Remove build artifacts + @rm -fr build/ + @rm -fr dist/ + @rm -fr .eggs/ + @find . -name '*.egg-info' -exec rm -fr {} + + @find . -name '*.egg' -exec rm -f {} + + +clean-pyc: # Remove Python file artifacts + @find . -name '*.pyc' -exec rm -f {} + + @find . -name '*.pyo' -exec rm -f {} + + @find . -name '*~' -exec rm -f {} + + @find . -name '__pycache__' -exec rm -fr {} + + +clean-test: # Remove test and coverage artifacts + @find . -name '.pytest_cache' -exec rm -fr {} + + @rm -fr .tox/ + @rm -f .coverage + @rm -fr htmlcov/ diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md new file mode 100644 index 0000000..b84b40b --- /dev/null +++ b/PYTHON_STYLE_GUIDE.md @@ -0,0 +1,97 @@ + + +# Python Style Guide + +This guide starts out with our general Python coding style guidelines and ends with a section on how we write & run (Python) tests. + +## General Python Coding Style Guidelines + +Our starting point is [PEP8](https://www.python.org/dev/peps/pep-0008/), the standard "Style Guide for Python Code." Many Python IDEs will check your code against PEP8. (Note that PEP8 isn't frozen; it actually changes over time, but slowly.) + +Planetmint uses Python 3.5+, so you can ignore all PEP8 guidelines specific to Python 2. + +We use [pre-commit](http://pre-commit.com/) to check some of the rules below before every commit but not everything is realized yet. +The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file. + +### Python Docstrings + +PEP8 says some things about docstrings, but not what to put in them or how to structure them. [PEP257](https://www.python.org/dev/peps/pep-0257/) was one proposal for docstring conventions, but we prefer [Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) instead: they're easier to read and the [napoleon extension](http://www.sphinx-doc.org/en/stable/ext/napoleon.html) for Sphinx lets us turn them into nice-looking documentation. Here are some references on Google-style docstrings: + +* [Google's docs on Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) +* [napoleon's docs include an overview of Google-style docstrings](http://sphinxcontrib-napoleon.readthedocs.org/en/latest/index.html) +* [Example Google-style docstrings](http://sphinxcontrib-napoleon.readthedocs.org/en/latest/example_google.html) (from napoleon's docs) + +### Maximum Line Length + +PEP8 has some [maximum line length guidelines](https://www.python.org/dev/peps/pep-0008/#id17), starting with "Limit all lines to a maximum of 79 characters" but "for flowing long blocks of text with fewer structural restrictions (docstrings or comments), the line length should be limited to 72 characters." + +We discussed this at length, and it seems that the consensus is: _try_ to keep line lengths less than 79/72 characters, unless you have a special situation where longer lines would improve readability. (The basic reason is that 79/72 works for everyone, and Planetmint is an open source project.) As a hard limit, keep all lines less than 119 characters (which is the width of GitHub code review). + +### Single or Double Quotes? + +Python lets you use single or double quotes. PEP8 says you can use either, as long as you're consistent. We try to stick to using single quotes, except in cases where using double quotes is more readable. For example: +```python +print('This doesn\'t look so nice.') +print("Doesn't this look nicer?") +``` + +### Breaking Strings Across Multiple Lines + +Should we use parentheses or slashes (`\`) to break strings across multiple lines, i.e. +```python +my_string = ('This is a very long string, so long that it will not fit into just one line ' + 'so it must be split across multiple lines.') +# or +my_string = 'This is a very long string, so long that it will not fit into just one line ' \ + 'so it must be split across multiple lines.' +``` + +It seems the preference is for slashes, but using parentheses is okay too. (There are good arguments either way. Arguing about it seems like a waste of time.) + +### How to Format Long import Statements + +If you need to `import` lots of names from a module or package, and they won't all fit in one line (without making the line too long), then use parentheses to spread the names across multiple lines, like so: +```python +from Tkinter import ( + Tk, Frame, Button, Entry, Canvas, Text, + LEFT, DISABLED, NORMAL, RIDGE, END, +) + +# Or + +from Tkinter import (Tk, Frame, Button, Entry, Canvas, Text, + LEFT, DISABLED, NORMAL, RIDGE, END) +``` + +For the rationale, see [PEP 328](https://www.python.org/dev/peps/pep-0328/#rationale-for-parentheses). + +### Using the % operator or `format()` to Format Strings + +Given the choice: +```python +x = 'name: %s; score: %d' % (name, n) +# or +x = 'name: {}; score: {}'.format(name, n) +``` + +we use the `format()` version. The [official Python documentation says](https://docs.python.org/2/library/stdtypes.html#str.format), "This method of string formatting is the new standard in Python 3, and should be preferred to the % formatting described in String Formatting Operations in new code." + + +## Running the Flake8 Style Checker + +We use [Flake8](http://flake8.pycqa.org/en/latest/index.html) to check our Python code style. Once you have it installed, you can run it using: +```text +flake8 --max-line-length 119 bigchaindb/ +``` + + +## Writing and Running (Python) Tests + +The content of this section was moved to [`bigchaindb/tests/README.md`](https://github.com/bigchaindb/bigchaindb/blob/master/tests/README.md). + +Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions. diff --git a/README.md b/README.md index 6ba72de..ac7b9b5 100644 --- a/README.md +++ b/README.md @@ -1 +1,77 @@ -# planetmint \ No newline at end of file + + + + +[![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master) +[![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases) +[![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/Planetmint/) +[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.com/bigchaindb/bigchaindb) +[![Documentation Status](https://readthedocs.org/projects/bigchaindb-server/badge/?version=latest)](https://docs.bigchaindb.com/projects/server/en/latest/) +[![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +# Planetmint Server + +Planetmint is the blockchain database. This repository is for _BigchainDB Server_. + +## The Basics + +* [Try the Quickstart](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html) +* [Read the Planetmint 2.0 whitepaper](https://www.bigchaindb.com/whitepaper/) +* [Check out the _Hitchiker's Guide to BigchainDB_](https://www.bigchaindb.com/developers/guide/) + +## Run and Test Planetmint Server from the `master` Branch + +Running and testing the latest version of Planetmint Server is easy. Make sure you have a recent version of [Docker Compose](https://docs.docker.com/compose/install/) installed. When you are ready, fire up a terminal and run: + +```text +git clone https://github.com/bigchaindb/bigchaindb.git +cd bigchaindb +make run +``` + +Planetmint should be reachable now on `http://localhost:9984/`. + +There are also other commands you can execute: + +* `make start`: Run Planetmint from source and daemonize it (stop it with `make stop`). +* `make stop`: Stop Planetmint. +* `make logs`: Attach to the logs. +* `make test`: Run all unit and acceptance tests. +* `make test-unit-watch`: Run all tests and wait. Every time you change code, tests will be run again. +* `make cov`: Check code coverage and open the result in the browser. +* `make doc`: Generate HTML documentation and open it in the browser. +* `make clean`: Remove all build, test, coverage and Python artifacts. +* `make reset`: Stop and REMOVE all containers. WARNING: you will LOSE all data stored in Planetmint. + +To view all commands available, run `make`. + +## Links for Everyone + +* [Planetmint.com](https://www.bigchaindb.com/) - the main Planetmint website, including newsletter signup +* [Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) +* [Blog](https://medium.com/the-bigchaindb-blog) +* [Twitter](https://twitter.com/Planetmint) + +## Links for Developers + +* [All Planetmint Documentation](https://docs.bigchaindb.com/en/latest/) +* [Planetmint Server Documentation](https://docs.bigchaindb.com/projects/server/en/latest/index.html) +* [CONTRIBUTING.md](.github/CONTRIBUTING.md) - how to contribute +* [Community guidelines](CODE_OF_CONDUCT.md) +* [Open issues](https://github.com/bigchaindb/bigchaindb/issues) +* [Open pull requests](https://github.com/bigchaindb/bigchaindb/pulls) +* [Gitter chatroom](https://gitter.im/bigchaindb/bigchaindb) + +## Legal + +* [Licenses](LICENSES.md) - open source & open content +* [Imprint](https://www.bigchaindb.com/imprint/) +* [Contact Us](https://www.bigchaindb.com/contact/) diff --git a/README_cn.md b/README_cn.md new file mode 100644 index 0000000..8385ace --- /dev/null +++ b/README_cn.md @@ -0,0 +1,77 @@ + + + + +[![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master) +[![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases) +[![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/Planetmint/) +[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.com/bigchaindb/bigchaindb) +[![Documentation Status](https://readthedocs.org/projects/bigchaindb-server/badge/?version=latest)](https://docs.bigchaindb.com/projects/server/en/latest/) +[![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +# Planetmint 服务器 + +Planetmint 是区块链数据库. 这是 _BigchainDB 服务器_ 的仓库. + +## 基础知识 + +* [尝试快速开始](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html) +* [阅读 Planetmint 2.0 白皮书](https://www.bigchaindb.com/whitepaper/) +* [查阅漫游指南](https://www.bigchaindb.com/developers/guide/) + +## 运行和测试 `master` 分支的 Planetmint 服务器 + +运行和测试最新版本的 Planetmint 服务器非常简单. 确认你有安装最新版本的 [Docker Compose](https://docs.docker.com/compose/install/). 当你准备好了, 打开一个终端并运行: + +```text +git clone https://github.com/bigchaindb/bigchaindb.git +cd bigchaindb +make run +``` + +Planetmint 应该可以通过 `http://localhost:9984/` 访问. + +这里也有一些其他的命令你可以运行: + +* `make start`: 通过源码和守护进程的方式运行 Planetmint (通过 `make stop` 停止). +* `make stop`: 停止运行 Planetmint. +* `make logs`: 附在日志上. +* `make test`: 运行所有单元和验收测试. +* `make test-unit-watch`: 运行所有测试并等待. 每次更改代码时都会再次运行测试. +* `make cov`: 检查代码覆盖率并在浏览器中打开结果. +* `make doc`: 生成 HTML 文档并在浏览器中打开它. +* `make clean`: 删除所有构建, 测试, 覆盖和 Python 生成物. +* `make reset`: 停止并移除所有容器. 警告: 您将丢失存储在 Planetmint 中的所有数据. + +查看所有可用命令, 请运行 `make`. + +## 一般人员链接 + +* [Planetmint.com](https://www.bigchaindb.com/) - Planetmint 主网站, 包括新闻订阅 +* [路线图](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) +* [博客](https://medium.com/the-bigchaindb-blog) +* [推特](https://twitter.com/Planetmint) + +## 开发人员链接 + +* [所有的 Planetmint 文档](https://docs.bigchaindb.com/en/latest/) +* [Planetmint 服务器 文档](https://docs.bigchaindb.com/projects/server/en/latest/index.html) +* [CONTRIBUTING.md](.github/CONTRIBUTING.md) - how to contribute +* [社区指南](CODE_OF_CONDUCT.md) +* [公开问题](https://github.com/bigchaindb/bigchaindb/issues) +* [公开的 pull request](https://github.com/bigchaindb/bigchaindb/pulls) +* [Gitter 聊天室](https://gitter.im/bigchaindb/bigchaindb) + +## 法律声明 + +* [许可](LICENSES.md) - 开源代码 & 开源内容 +* [印记](https://www.bigchaindb.com/imprint/) +* [联系我们](https://www.bigchaindb.com/contact/) diff --git a/README_kor.md b/README_kor.md new file mode 100644 index 0000000..c4f440f --- /dev/null +++ b/README_kor.md @@ -0,0 +1,65 @@ +[![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master) +[![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases) +[![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/Planetmint/) +[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.org/bigchaindb/bigchaindb) +[![Documentation Status](https://readthedocs.org/projects/bigchaindb-server/badge/?version=latest)](https://docs.bigchaindb.com/projects/server/en/latest/) +[![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +# Planetmint 서버 + +BigchaingDB는 블록체인 데이터베이스입니다. 이 저장소는 _BigchaingDB 서버_를 위한 저장소입니다. + +### 기본 사항 + +* [빠른 시작 사용해보기](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html) +* [Planetmint 2.0 백서 읽기](https://www.bigchaindb.com/whitepaper/) +* [BigchainDB에 대한 _Hitchiker's Guide_를 확인십시오.](https://www.bigchaindb.com/developers/guide/) + +### `master` Branch에서 Planetmint 서버 실행 및 테스트 + +BigchaingDB 서버의 최신 버전을 실행하고 테스트하는 것은 어렵지 않습니다. [Docker Compose](https://docs.docker.com/compose/install/)의 최신 버전이 설치되어 있는지 확인하십시오. 준비가 되었다면, 터미널에서 다음을 실행하십시오. + +```text +git clone https://github.com/bigchaindb/bigchaindb.git +cd bigchaindb +make run +``` + +이제 BigchainDB는 `http://localhost:9984/`에 연결되어야 합니다. + +또한, 실행시키기 위한 다른 명령어들도 있습니다. + +* `make start` : 소스로부터 BigchainDB를 실행하고 데몬화합니다. \(이는 `make stop` 을 하면 중지합니다.\) +* `make stop` : BigchainDB를 중지합니다. +* `make logs` : 로그에 첨부합니다. +* `make text` : 모든 유닛과 허가 테스트를 실행합니다. +* `make test-unit-watch` : 모든 테스트를 수행하고 기다립니다. 코드를 변경할 때마다 테스트는 다시 실행될 것입니다. +* `make cov` : 코드 커버리지를 확인하고 브라우저에서 결과를 엽니다. +* `make doc` : HTML 문서를 만들고, 브라우저에서 엽니다. +* `make clean` : 모든 빌드와 테스트, 커버리지 및 파이썬 아티팩트를 제거합니다. +* `make reset` : 모든 컨테이너들을 중지하고 제거합니다. 경고 : BigchainDB에 저장된 모든 데이터를 잃을 수 있습니다. + +사용 가능한 모든 명령어를 보기 위해서는 `make` 를 실행하십시오. + +### 모두를 위한 링크들 + +* [Planetmint.com ](https://www.bigchaindb.com/)- 뉴스 레터 가입을 포함하는 Planetmint 주요 웹 사이트 +* [로드맵](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) +* [블로그](https://medium.com/the-bigchaindb-blog) +* [트위터](https://twitter.com/Planetmint) + +### 개발자들을 위한 링크들 + +* [모든 Planetmint 문서](https://docs.bigchaindb.com/en/latest/) +* [Planetmint 서버 문서](https://docs.bigchaindb.com/projects/server/en/latest/index.html) +* [CONTRIBUTING.md](https://github.com/bigchaindb/bigchaindb/blob/master/.github/CONTRIBUTING.md) - 기여를 하는 방법 +* [커뮤니티 가이드라인](https://github.com/bigchaindb/bigchaindb/blob/master/CODE_OF_CONDUCT.md) +* [이슈 작성](https://github.com/bigchaindb/bigchaindb/issues) +* [pull request 하기](https://github.com/bigchaindb/bigchaindb/pulls) +* [Gitter 채팅방](https://gitter.im/bigchaindb/bigchaindb) + +### 합법 + +* [라이선스](https://github.com/bigchaindb/bigchaindb/blob/master/LICENSES.md) - 오픈 소스 & 오픈 콘텐츠 +* [발행](https://www.bigchaindb.com/imprint/) +* [연락처](https://www.bigchaindb.com/contact/) diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md new file mode 100644 index 0000000..117e407 --- /dev/null +++ b/RELEASE_PROCESS.md @@ -0,0 +1,101 @@ + + +# Our Release Process + +## Notes + +Planetmint follows +[the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) +(i.e. MAJOR.MINOR.PATCH), +which is almost identical +to [regular semantic versioning](http://semver.org/), but there's no hyphen, e.g. + +- `0.9.0` for a typical final release +- `4.5.2a1` not `4.5.2-a1` for the first Alpha release +- `3.4.5rc2` not `3.4.5-rc2` for Release Candidate 2 + +**Note 1:** For Git tags (which are used to identify releases on GitHub), we append a `v` in front. For example, the Git tag for version `2.0.0a1` was `v2.0.0a1`. + +**Note 2:** For Docker image tags (e.g. on Docker Hub), we use longer version names for Alpha, Beta and Release Candidate releases. For example, the Docker image tag for version `2.0.0a2` was `2.0.0-alpha2`. + +We use `0.9` and `0.9.0` as example version and short-version values below. You should replace those with the correct values for your new version. + +We follow [BEP-1](https://github.com/bigchaindb/BEPs/tree/master/1), which is our variant of C4, the Collective Code Construction Contract, so a release is just a [tagged commit](https://git-scm.com/book/en/v2/Git-Basics-Tagging) on the `master` branch, i.e. a label for a particular Git commit. + +The following steps are what we do to release a new version of _BigchainDB Server_. The steps to release the Python Driver are similar but not the same. + +## Steps + +1. Create a pull request where you make the following changes: + + - Update `CHANGELOG.md` + - Update all Docker image tags in all Kubernetes YAML files (in the `k8s/` directory). + For example, in the files: + + - `k8s/bigchaindb/bigchaindb-ss.yaml` and + - `k8s/dev-setup/bigchaindb.yaml` + + find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.) + Keep in mind that this is a _Docker image tag_ so our naming convention is + a bit different; see Note 2 in the **Notes** section above. + - In `bigchaindb/version.py`: + - update `__version__` to e.g. `0.9.0` (with no `.dev` on the end) + - update `__short_version__` to e.g. `0.9` (with no `.dev` on the end) + - In the docs about installing Planetmint (and Tendermint), and in the associated scripts, recommend/install a version of Tendermint that _actually works_ with the soon-to-be-released version of Planetmint. You can find all such references by doing a search for the previously-recommended version number, such as `0.31.5`. + - In `setup.py`, _maybe_ update the development status item in the `classifiers` list. For example, one allowed value is `"Development Status :: 5 - Production/Stable"`. The [allowed values are listed at pypi.python.org](https://pypi.python.org/pypi?%3Aaction=list_classifiers). + +2. **Wait for all the tests to pass!** +3. Merge the pull request into the `master` branch. +4. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) + and click the "Draft a new release" button. +5. Fill in the details: + - **Tag version:** version number preceded by `v`, e.g. `v0.9.1` + - **Target:** the last commit that was just merged. In other words, that commit will get a Git tag with the value given for tag version above. + - **Title:** Same as tag version above, e.g `v0.9.1` + - **Description:** The body of the changelog entry (Added, Changed, etc.) +6. Click "Publish release" to publish the release on GitHub. +7. On your local computer, make sure you're on the `master` branch and that it's up-to-date with the `master` branch in the bigchaindb/bigchaindb repository (e.g. `git pull upstream master`). We're going to use that to push a new `bigchaindb` package to PyPI. +8. Make sure you have a `~/.pypirc` file containing credentials for PyPI. +9. Do `make release` to build and publish the new `bigchaindb` package on PyPI. For this step you need to have `twine` installed. If you get an error like `Makefile:135: recipe for target 'clean-pyc' failed` then try doing + ```text + sudo chown -R $(whoami):$(whoami) . + ``` +10. [Log in to readthedocs.org](https://readthedocs.org/accounts/login/) and go to the **Planetmint Server** project, then: + - Click on "Builds", select "latest" from the drop-down menu, then click the "Build Version:" button. + - Wait for the build of "latest" to finish. This can take a few minutes. + - Go to Admin --> Advanced Settings + and make sure that "Default branch:" (i.e. what "latest" points to) + is set to the new release's tag, e.g. `v0.9.1`. + (It won't be an option if you didn't wait for the build of "latest" to finish.) + Then scroll to the bottom and click "Save". + - Go to Admin --> Versions + and under **Choose Active Versions**, do these things: + 1. Make sure that the new version's tag is "Active" and "Public" + 2. Make sure the **stable** branch is _not_ active. + 3. Scroll to the bottom of the page and click "Save". +11. Go to [Docker Hub](https://hub.docker.com/) and sign in, then: + - Click on "Organizations" + - Click on "bigchaindb" + - Click on "bigchaindb/bigchaindb" + - Click on "Build Settings" + - Find the row where "Docker Tag Name" equals `latest` + and change the value of "Name" to the name (Git tag) + of the new release, e.g. `v0.9.0`. + - If the release is an Alpha, Beta or Release Candidate release, + then a new row must be added. + You can do that by clicking the green "+" (plus) icon. + The contents of the new row should be similar to the existing rows + of previous releases like that. + - Click on "Tags" + - Delete the "latest" tag (so we can rebuild it) + - Click on "Build Settings" again + - Click on the "Trigger" button for the "latest" tag and make sure it worked by clicking on "Tags" again + - If the release is an Alpha, Beta or Release Candidate release, + then click on the "Trigger" button for that tag as well. + +Congratulations, you have released a new version of Planetmint Server! diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..ba00fd0 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,14 @@ + + +# Planetmint Roadmap + +We moved the Planetmint Roadmap to the bigchaindb/org repository; see: + +[https://github.com/bigchaindb/org/blob/master/ROADMAP.md](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) + +(We kept this file here to avoid breaking some links.) diff --git a/acceptance/README.md b/acceptance/README.md new file mode 100644 index 0000000..56264d3 --- /dev/null +++ b/acceptance/README.md @@ -0,0 +1,27 @@ + + +# Acceptance test suite +This directory contains the acceptance test suite for Planetmint. + +The suite uses Docker Compose to set up a single Planetmint node, run all tests, and finally stop the node. In the future we will add support for a four node network setup. + +## Running the tests +It should be as easy as `make test-acceptance`. + +Note that `make test-acceptance` will take some time to start the node and shutting it down. If you are developing a test, or you wish to run a specific test in the acceptance test suite, first start the node with `make start`. After the node is running, you can run `pytest` inside the `python-acceptance` container with: + +```bash +docker-compose run --rm python-acceptance pytest +``` + +## Writing and documenting the tests +Tests are sometimes difficult to read. For acceptance tests, we try to be really explicit on what the test is doing, so please write code that is *simple* and easy to understand. We decided to use literate-programming documentation. To generate the documentation run: + +```bash +make doc-acceptance +``` diff --git a/acceptance/python/.gitignore b/acceptance/python/.gitignore new file mode 100644 index 0000000..d8f8d46 --- /dev/null +++ b/acceptance/python/.gitignore @@ -0,0 +1 @@ +docs diff --git a/acceptance/python/Dockerfile b/acceptance/python/Dockerfile new file mode 100644 index 0000000..144fe64 --- /dev/null +++ b/acceptance/python/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.6.3 + +RUN mkdir -p /src +RUN pip install --upgrade \ + pycco \ + websocket-client~=0.47.0 \ + pytest~=3.0 \ + bigchaindb-driver~=0.6.2 \ + blns diff --git a/acceptance/python/src/test_basic.py b/acceptance/python/src/test_basic.py new file mode 100644 index 0000000..263dc1b --- /dev/null +++ b/acceptance/python/src/test_basic.py @@ -0,0 +1,125 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# # Basic Acceptance Test +# Here we check that the primitives of the system behave as expected. +# As you will see, this script tests basic stuff like: +# +# - create a transaction +# - check if the transaction is stored +# - check for the outputs of a given public key +# - transfer the transaction to another key +# +# We run a series of checks for each steps, that is retrieving the transaction from +# the remote system, and also checking the `outputs` of a given public key. +# +# This acceptance test is a rip-off of our +# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html). + +# ## Imports +# We need some utils from the `os` package, we will interact with +# env variables. +import os + +# For this test case we import and use the Python Driver. +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair + + +def test_basic(): + # ## Set up a connection to Planetmint + # To use BighainDB we need a connection. Here we create one. By default we + # connect to localhost, but you can override this value using the env variable + # called `PLANETMINT_ENDPOINT`, a valid value must include the schema: + # `https://example.com:9984` + bdb = BigchainDB(os.environ.get('PLANETMINT_ENDPOINT')) + + # ## Create keypairs + # This test requires the interaction between two actors with their own keypair. + # The two keypairs will be called—drum roll—Alice and Bob. + alice, bob = generate_keypair(), generate_keypair() + + # ## Alice registers her bike in Planetmint + # Alice has a nice bike, and here she creates the "digital twin" + # of her bike. + bike = {'data': {'bicycle': {'serial_number': 420420}}} + + # She prepares a `CREATE` transaction... + prepared_creation_tx = bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + asset=bike) + + # ... and she fulfills it with her private key. + fulfilled_creation_tx = bdb.transactions.fulfill( + prepared_creation_tx, + private_keys=alice.private_key) + + # We will use the `id` of this transaction several time, so we store it in + # a variable with a short and easy name + bike_id = fulfilled_creation_tx['id'] + + # Now she is ready to send it to the Planetmint Network. + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_creation_tx) + + # And just to be 100% sure, she also checks if she can retrieve + # it from the Planetmint node. + assert bdb.transactions.retrieve(bike_id), 'Cannot find transaction {}'.format(bike_id) + + # Alice is now the proud owner of one unspent asset. + assert len(bdb.outputs.get(alice.public_key, spent=False)) == 1 + assert bdb.outputs.get(alice.public_key)[0]['transaction_id'] == bike_id + + # ## Alice transfers her bike to Bob + # After registering her bike, Alice is ready to transfer it to Bob. + # She needs to create a new `TRANSFER` transaction. + + # A `TRANSFER` transaction contains a pointer to the original asset. The original asset + # is identified by the `id` of the `CREATE` transaction that defined it. + transfer_asset = {'id': bike_id} + + # Alice wants to spend the one and only output available, the one with index `0`. + output_index = 0 + output = fulfilled_creation_tx['outputs'][output_index] + + # Here, she defines the `input` of the `TRANSFER` transaction. The `input` contains + # several keys: + # + # - `fulfillment`, taken from the previous `CREATE` transaction. + # - `fulfills`, that specifies which condition she is fulfilling. + # - `owners_before`. + transfer_input = {'fulfillment': output['condition']['details'], + 'fulfills': {'output_index': output_index, + 'transaction_id': fulfilled_creation_tx['id']}, + 'owners_before': output['public_keys']} + + # Now that all the elements are set, she creates the actual transaction... + prepared_transfer_tx = bdb.transactions.prepare( + operation='TRANSFER', + asset=transfer_asset, + inputs=transfer_input, + recipients=bob.public_key) + + # ... and signs it with her private key. + fulfilled_transfer_tx = bdb.transactions.fulfill( + prepared_transfer_tx, + private_keys=alice.private_key) + + # She finally sends the transaction to a Planetmint node. + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx) + + # And just to be 100% sure, she also checks if she can retrieve + # it from the Planetmint node. + assert bdb.transactions.retrieve(fulfilled_transfer_tx['id']) == sent_transfer_tx + + # Now Alice has zero unspent transactions. + assert len(bdb.outputs.get(alice.public_key, spent=False)) == 0 + + # While Bob has one. + assert len(bdb.outputs.get(bob.public_key, spent=False)) == 1 + + # Bob double checks what he got was the actual bike. + bob_tx_id = bdb.outputs.get(bob.public_key, spent=False)[0]['transaction_id'] + assert bdb.transactions.retrieve(bob_tx_id) == sent_transfer_tx diff --git a/acceptance/python/src/test_divisible_asset.py b/acceptance/python/src/test_divisible_asset.py new file mode 100644 index 0000000..d02ac2b --- /dev/null +++ b/acceptance/python/src/test_divisible_asset.py @@ -0,0 +1,181 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# # Divisible assets integration testing +# This test checks if we can successfully divide assets. +# The script tests various things like: +# +# - create a transaction with a divisible asset and issue them to someone +# - check if the transaction is stored and has the right amount of tokens +# - spend some tokens +# - try to spend more tokens than available +# +# We run a series of checks for each step, that is retrieving +# the transaction from the remote system, and also checking the `amount` +# of a given transaction. +# +# This integration test is a rip-off of our +# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html). + +# ## Imports +# We need some utils from the `os` package, we will interact with +# env variables. +# We need the `pytest` package to catch the `BadRequest` exception properly. +# And of course, we also need the `BadRequest`. +import os +import pytest +from bigchaindb_driver.exceptions import BadRequest + +# For this test case we import and use the Python Driver. +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair + + +def test_divisible_assets(): + # ## Set up a connection to Planetmint + # Check [test_basic.py](./test_basic.html) to get some more details + # about the endpoint. + bdb = BigchainDB(os.environ.get('PLANETMINT_ENDPOINT')) + + # Oh look, it is Alice again and she brought her friend Bob along. + alice, bob = generate_keypair(), generate_keypair() + + # ## Alice creates a time sharing token + # Alice wants to go on vacation, while Bobs bike just broke down. + # Alice decides to rent her bike to Bob while she is gone. + # So she prepares a `CREATE` transaction to issues 10 tokens. + # First, she prepares an asset for a time sharing token. As you can see in + # the description, Bob and Alice agree that each token can be used to ride + # the bike for one hour. + + bike_token = { + 'data': { + 'token_for': { + 'bike': { + 'serial_number': 420420 + } + }, + 'description': 'Time share token. Each token equals one hour of riding.', + }, + } + + # She prepares a `CREATE` transaction and issues 10 tokens. + # Here, Alice defines in a tuple that she wants to assign + # these 10 tokens to Bob. + prepared_token_tx = bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + recipients=[([bob.public_key], 10)], + asset=bike_token) + + # She fulfills and sends the transaction. + fulfilled_token_tx = bdb.transactions.fulfill( + prepared_token_tx, + private_keys=alice.private_key) + + bdb.transactions.send_commit(fulfilled_token_tx) + + # We store the `id` of the transaction to use it later on. + bike_token_id = fulfilled_token_tx['id'] + + # Let's check if the transaction was successful. + assert bdb.transactions.retrieve(bike_token_id), \ + 'Cannot find transaction {}'.format(bike_token_id) + + # Bob owns 10 tokens now. + assert bdb.transactions.retrieve(bike_token_id)['outputs'][0][ + 'amount'] == '10' + + # ## Bob wants to use the bike + # Now that Bob got the tokens and the sun is shining, he wants to get out + # with the bike for three hours. + # To use the bike he has to send the tokens back to Alice. + # To learn about the details of transferring a transaction check out + # [test_basic.py](./test_basic.html) + transfer_asset = {'id': bike_token_id} + + output_index = 0 + output = fulfilled_token_tx['outputs'][output_index] + transfer_input = {'fulfillment': output['condition']['details'], + 'fulfills': {'output_index': output_index, + 'transaction_id': fulfilled_token_tx[ + 'id']}, + 'owners_before': output['public_keys']} + + # To use the tokens Bob has to reassign 7 tokens to himself and the + # amount he wants to use to Alice. + prepared_transfer_tx = bdb.transactions.prepare( + operation='TRANSFER', + asset=transfer_asset, + inputs=transfer_input, + recipients=[([alice.public_key], 3), ([bob.public_key], 7)]) + + # He signs and sends the transaction. + fulfilled_transfer_tx = bdb.transactions.fulfill( + prepared_transfer_tx, + private_keys=bob.private_key) + + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx) + + # First, Bob checks if the transaction was successful. + assert bdb.transactions.retrieve( + fulfilled_transfer_tx['id']) == sent_transfer_tx + # There are two outputs in the transaction now. + # The first output shows that Alice got back 3 tokens... + assert bdb.transactions.retrieve( + fulfilled_transfer_tx['id'])['outputs'][0]['amount'] == '3' + + # ... while Bob still has 7 left. + assert bdb.transactions.retrieve( + fulfilled_transfer_tx['id'])['outputs'][1]['amount'] == '7' + + # ## Bob wants to ride the bike again + # It's been a week and Bob wants to right the bike again. + # Now he wants to ride for 8 hours, that's a lot Bob! + # He prepares the transaction again. + + transfer_asset = {'id': bike_token_id} + # This time we need an `output_index` of 1, since we have two outputs + # in the `fulfilled_transfer_tx` we created before. The first output with + # index 0 is for Alice and the second output is for Bob. + # Since Bob wants to spend more of his tokens he has to provide the + # correct output with the correct amount of tokens. + output_index = 1 + + output = fulfilled_transfer_tx['outputs'][output_index] + + transfer_input = {'fulfillment': output['condition']['details'], + 'fulfills': {'output_index': output_index, + 'transaction_id': fulfilled_transfer_tx['id']}, + 'owners_before': output['public_keys']} + + # This time Bob only provides Alice in the `recipients` because he wants + # to spend all his tokens + prepared_transfer_tx = bdb.transactions.prepare( + operation='TRANSFER', + asset=transfer_asset, + inputs=transfer_input, + recipients=[([alice.public_key], 8)]) + + fulfilled_transfer_tx = bdb.transactions.fulfill( + prepared_transfer_tx, + private_keys=bob.private_key) + + # Oh Bob, what have you done?! You tried to spend more tokens than you had. + # Remember Bob, last time you spent 3 tokens already, + # so you only have 7 left. + with pytest.raises(BadRequest) as error: + bdb.transactions.send_commit(fulfilled_transfer_tx) + + # Now Bob gets an error saying that the amount he wanted to spent is + # higher than the amount of tokens he has left. + assert error.value.args[0] == 400 + message = 'Invalid transaction (AmountError): The amount used in the ' \ + 'inputs `7` needs to be same as the amount used in the ' \ + 'outputs `8`' + assert error.value.args[2]['message'] == message + + # We have to stop this test now, I am sorry, but Bob is pretty upset + # about his mistake. See you next time :) diff --git a/acceptance/python/src/test_double_spend.py b/acceptance/python/src/test_double_spend.py new file mode 100644 index 0000000..66a2467 --- /dev/null +++ b/acceptance/python/src/test_double_spend.py @@ -0,0 +1,48 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# # Double Spend testing +# This test challenge the system with double spends. + +import os +from uuid import uuid4 +from threading import Thread +import queue + +import bigchaindb_driver.exceptions +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair + + +def test_double_create(): + bdb = BigchainDB(os.environ.get('PLANETMINT_ENDPOINT')) + alice = generate_keypair() + + results = queue.Queue() + + tx = bdb.transactions.fulfill( + bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + asset={'data': {'uuid': str(uuid4())}}), + private_keys=alice.private_key) + + def send_and_queue(tx): + try: + bdb.transactions.send_commit(tx) + results.put('OK') + except bigchaindb_driver.exceptions.TransportError as e: + results.put('FAIL') + + t1 = Thread(target=send_and_queue, args=(tx, )) + t2 = Thread(target=send_and_queue, args=(tx, )) + + t1.start() + t2.start() + + results = [results.get(timeout=2), results.get(timeout=2)] + + assert results.count('OK') == 1 + assert results.count('FAIL') == 1 diff --git a/acceptance/python/src/test_multiple_owners.py b/acceptance/python/src/test_multiple_owners.py new file mode 100644 index 0000000..f407c7d --- /dev/null +++ b/acceptance/python/src/test_multiple_owners.py @@ -0,0 +1,126 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# # Multiple owners integration testing +# This test checks if we can successfully create and transfer a transaction +# with multiple owners. +# The script tests various things like: +# +# - create a transaction with multiple owners +# - check if the transaction is stored and has the right amount of public keys +# - transfer the transaction to a third person +# +# We run a series of checks for each step, that is retrieving +# the transaction from the remote system, and also checking the public keys +# of a given transaction. +# +# This integration test is a rip-off of our +# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html). + +# ## Imports +# We need some utils from the `os` package, we will interact with +# env variables. +import os + +# For this test case we import and use the Python Driver. +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair + + +def test_multiple_owners(): + # ## Set up a connection to Planetmint + # Check [test_basic.py](./test_basic.html) to get some more details + # about the endpoint. + bdb = BigchainDB(os.environ.get('PLANETMINT_ENDPOINT')) + + # Hey Alice and Bob, nice to see you again! + alice, bob = generate_keypair(), generate_keypair() + + # ## Alice and Bob create a transaction + # Alice and Bob just moved into a shared flat, no one can afford these + # high rents anymore. Bob suggests to get a dish washer for the + # kitchen. Alice agrees and here they go, creating the asset for their + # dish washer. + dw_asset = { + 'data': { + 'dish washer': { + 'serial_number': 1337 + } + } + } + + # They prepare a `CREATE` transaction. To have multiple owners, both + # Bob and Alice need to be the recipients. + prepared_dw_tx = bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + recipients=(alice.public_key, bob.public_key), + asset=dw_asset) + + # Now they both sign the transaction by providing their private keys. + # And send it afterwards. + fulfilled_dw_tx = bdb.transactions.fulfill( + prepared_dw_tx, + private_keys=[alice.private_key, bob.private_key]) + + bdb.transactions.send_commit(fulfilled_dw_tx) + + # We store the `id` of the transaction to use it later on. + dw_id = fulfilled_dw_tx['id'] + + # Let's check if the transaction was successful. + assert bdb.transactions.retrieve(dw_id), \ + 'Cannot find transaction {}'.format(dw_id) + + # The transaction should have two public keys in the outputs. + assert len( + bdb.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2 + + # ## Alice and Bob transfer a transaction to Carol. + # Alice and Bob save a lot of money living together. They often go out + # for dinner and don't cook at home. But now they don't have any dishes to + # wash, so they decide to sell the dish washer to their friend Carol. + + # Hey Carol, nice to meet you! + carol = generate_keypair() + + # Alice and Bob prepare the transaction to transfer the dish washer to + # Carol. + transfer_asset = {'id': dw_id} + + output_index = 0 + output = fulfilled_dw_tx['outputs'][output_index] + transfer_input = {'fulfillment': output['condition']['details'], + 'fulfills': {'output_index': output_index, + 'transaction_id': fulfilled_dw_tx[ + 'id']}, + 'owners_before': output['public_keys']} + + # Now they create the transaction... + prepared_transfer_tx = bdb.transactions.prepare( + operation='TRANSFER', + asset=transfer_asset, + inputs=transfer_input, + recipients=carol.public_key) + + # ... and sign it with their private keys, then send it. + fulfilled_transfer_tx = bdb.transactions.fulfill( + prepared_transfer_tx, + private_keys=[alice.private_key, bob.private_key]) + + sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx) + + # They check if the transaction was successful. + assert bdb.transactions.retrieve( + fulfilled_transfer_tx['id']) == sent_transfer_tx + + # The owners before should include both Alice and Bob. + assert len( + bdb.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][ + 'owners_before']) == 2 + + # While the new owner is Carol. + assert bdb.transactions.retrieve(fulfilled_transfer_tx['id'])[ + 'outputs'][0]['public_keys'][0] == carol.public_key diff --git a/acceptance/python/src/test_naughty_strings.py b/acceptance/python/src/test_naughty_strings.py new file mode 100644 index 0000000..f0e92d4 --- /dev/null +++ b/acceptance/python/src/test_naughty_strings.py @@ -0,0 +1,101 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# ## Testing potentially hazardous strings +# This test uses a library of `naughty` strings (code injections, weird unicode chars., etc.) as both keys and values. +# We look for either a successful tx, or in the case that we use a naughty string as a key, and it violates some key +# constraints, we expect to receive a well formatted error message. + +# ## Imports +# We need some utils from the `os` package, we will interact with +# env variables. +import os + +# Since the naughty strings get encoded and decoded in odd ways, +# we'll use a regex to sweep those details under the rug. +import re + +# We'll use a nice library of naughty strings... +from blns import blns + +# And parameterize our test so each one is treated as a separate test case +import pytest + +# For this test case we import and use the Python Driver. +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair +from bigchaindb_driver.exceptions import BadRequest + +naughty_strings = blns.all() + + +# This is our base test case, but we'll reuse it to send naughty strings as both keys and values. +def send_naughty_tx(asset, metadata): + # ## Set up a connection to Planetmint + # Check [test_basic.py](./test_basic.html) to get some more details + # about the endpoint. + bdb = BigchainDB(os.environ.get('PLANETMINT_ENDPOINT')) + + # Here's Alice. + alice = generate_keypair() + + # Alice is in a naughty mood today, so she creates a tx with some naughty strings + prepared_transaction = bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + asset=asset, + metadata=metadata) + + # She fulfills the transaction + fulfilled_transaction = bdb.transactions.fulfill( + prepared_transaction, + private_keys=alice.private_key) + + # The fulfilled tx gets sent to the BDB network + try: + sent_transaction = bdb.transactions.send_commit(fulfilled_transaction) + except BadRequest as e: + sent_transaction = e + + # If her key contained a '.', began with a '$', or contained a NUL character + regex = '.*\..*|\$.*|.*\x00.*' + key = next(iter(metadata)) + if re.match(regex, key): + # Then she expects a nicely formatted error code + status_code = sent_transaction.status_code + error = sent_transaction.error + regex = ( + r'\{\s*\n*' + r'\s*"message":\s*"Invalid transaction \(ValidationError\):\s*' + r'Invalid key name.*The key name cannot contain characters.*\n*' + r'\s*"status":\s*400\n*' + r'\s*\}\n*') + assert status_code == 400 + assert re.fullmatch(regex, error), sent_transaction + # Otherwise, she expects to see her transaction in the database + elif 'id' in sent_transaction.keys(): + tx_id = sent_transaction['id'] + assert bdb.transactions.retrieve(tx_id) + # If neither condition was true, then something weird happened... + else: + raise TypeError(sent_transaction) + + +@pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings) +def test_naughty_keys(naughty_string): + + asset = {'data': {naughty_string: 'nice_value'}} + metadata = {naughty_string: 'nice_value'} + + send_naughty_tx(asset, metadata) + + +@pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings) +def test_naughty_values(naughty_string): + + asset = {'data': {'nice_key': naughty_string}} + metadata = {'nice_key': naughty_string} + + send_naughty_tx(asset, metadata) diff --git a/acceptance/python/src/test_stream.py b/acceptance/python/src/test_stream.py new file mode 100644 index 0000000..c82f43d --- /dev/null +++ b/acceptance/python/src/test_stream.py @@ -0,0 +1,132 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# # Stream Acceptance Test +# This test checks if the event stream works correctly. The basic idea of this +# test is to generate some random **valid** transaction, send them to a +# Planetmint node, and expect those transactions to be returned by the valid +# transactions Stream API. During this test, two threads work together, +# sharing a queue to exchange events. +# +# - The *main thread* first creates and sends the transactions to Planetmint; +# then it run through all events in the shared queue to check if all +# transactions sent have been validated by Planetmint. +# - The *listen thread* listens to the events coming from Planetmint and puts +# them in a queue shared with the main thread. + +import os +import queue +import json +from threading import Thread, Event +from uuid import uuid4 + +# For this script, we need to set up a websocket connection, that's the reason +# we import the +# [websocket](https://github.com/websocket-client/websocket-client) module +from websocket import create_connection + +from bigchaindb_driver import BigchainDB +from bigchaindb_driver.crypto import generate_keypair + + +def test_stream(): + # ## Set up the test + # We use the env variable `BICHAINDB_ENDPOINT` to know where to connect. + # Check [test_basic.py](./test_basic.html) for more information. + BDB_ENDPOINT = os.environ.get('PLANETMINT_ENDPOINT') + + # *That's pretty bad, but let's do like this for now.* + WS_ENDPOINT = 'ws://{}:9985/api/v1/streams/valid_transactions'.format(BDB_ENDPOINT.rsplit(':')[0]) + + bdb = BigchainDB(BDB_ENDPOINT) + + # Hello to Alice again, she is pretty active in those tests, good job + # Alice! + alice = generate_keypair() + + # We need few variables to keep the state, specifically we need `sent` to + # keep track of all transactions Alice sent to Planetmint, while `received` + # are the transactions Planetmint validated and sent back to her. + sent = [] + received = queue.Queue() + + # In this test we use a websocket. The websocket must be started **before** + # sending transactions to Planetmint, otherwise we might lose some + # transactions. The `ws_ready` event is used to synchronize the main thread + # with the listen thread. + ws_ready = Event() + + # ## Listening to events + # This is the function run by the complementary thread. + def listen(): + # First we connect to the remote endpoint using the WebSocket protocol. + ws = create_connection(WS_ENDPOINT) + + # After the connection has been set up, we can signal the main thread + # to proceed (continue reading, it should make sense in a second.) + ws_ready.set() + + # It's time to consume all events coming from the Planetmint stream API. + # Every time a new event is received, it is put in the queue shared + # with the main thread. + while True: + result = ws.recv() + received.put(result) + + # Put `listen` in a thread, and start it. Note that `listen` is a local + # function and it can access all variables in the enclosing function. + t = Thread(target=listen, daemon=True) + t.start() + + # ## Pushing the transactions to Planetmint + # After starting the listen thread, we wait for it to connect, and then we + # proceed. + ws_ready.wait() + + # Here we prepare, sign, and send ten different `CREATE` transactions. To + # make sure each transaction is different from the other, we generate a + # random `uuid`. + for _ in range(10): + tx = bdb.transactions.fulfill( + bdb.transactions.prepare( + operation='CREATE', + signers=alice.public_key, + asset={'data': {'uuid': str(uuid4())}}), + private_keys=alice.private_key) + # We don't want to wait for each transaction to be in a block. By using + # `async` mode, we make sure that the driver returns as soon as the + # transaction is pushed to the Planetmint API. Remember: we expect all + # transactions to be in the shared queue: this is a two phase test, + # first we send a bunch of transactions, then we check if they are + # valid (and, in this case, they should). + bdb.transactions.send_async(tx) + + # The `id` of every sent transaction is then stored in a list. + sent.append(tx['id']) + + # ## Check the valid transactions coming from Planetmint + # Now we are ready to check if Planetmint did its job. A simple way to + # check if all sent transactions have been processed is to **remove** from + # `sent` the transactions we get from the *listen thread*. At one point in + # time, `sent` should be empty, and we exit the test. + while sent: + # To avoid waiting forever, we have an arbitrary timeout of 5 + # seconds: it should be enough time for Planetmint to create + # blocks, in fact a new block is created every second. If we hit + # the timeout, then game over ¯\\\_(ツ)\_/¯ + try: + event = received.get(timeout=5) + txid = json.loads(event)['transaction_id'] + except queue.Empty: + assert False, 'Did not receive all expected transactions' + + # Last thing is to try to remove the `txid` from the set of sent + # transactions. If this test is running in parallel with others, we + # might get a transaction id of another test, and `remove` can fail. + # It's OK if this happens. + try: + sent.remove(txid) + except ValueError: + pass diff --git a/bigchaindb/README.md b/bigchaindb/README.md new file mode 100644 index 0000000..9988929 --- /dev/null +++ b/bigchaindb/README.md @@ -0,0 +1,42 @@ + + +# Overview + +A high-level description of the files and subdirectories of Planetmint. + +## Files + +### [`lib.py`](lib.py) + +The `Planetmint` class is defined here. Most node-level operations and database interactions are found in this file. This is the place to start if you are interested in implementing a server API, since many of these class methods concern Planetmint interacting with the outside world. + +### [`models.py`](./models.py) + +`Block`, `Transaction`, and `Asset` classes are defined here. The classes mirror the block and transaction structure from the [documentation](https://docs.bigchaindb.com/projects/server/en/latest/data-models/index.html), but also include methods for validation and signing. + +### [`validation.py`](./validation.py) + +Base class for validation methods (verification of votes, blocks, and transactions). The actual logic is mostly found in `transaction` and `block` models, defined in [`models.py`](./models.py). + +### [`processes.py`](./processes.py) + +Entry point for the Planetmint process, after initialization. All subprocesses are started here: processes to handle new blocks, votes, etc. + +### [`config_utils.py`](./config_utils.py) + +Methods for managing the configuration, including loading configuration files, automatically generating the configuration, and keeping the configuration consistent across Planetmint instances. + +## Folders + +### [`commands`](./commands) + +Contains code for the [CLI](https://docs.bigchaindb.com/projects/server/en/latest/server-reference/bigchaindb-cli.html) for Planetmint. + +### [`db`](./db) + +Code for building the database connection, creating indexes, and other database setup tasks. diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py new file mode 100644 index 0000000..6bba680 --- /dev/null +++ b/bigchaindb/__init__.py @@ -0,0 +1,106 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import copy +import logging + +from bigchaindb.log import DEFAULT_LOGGING_CONFIG as log_config +from bigchaindb.lib import BigchainDB # noqa +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection +from bigchaindb.version import __version__ # noqa +from bigchaindb.core import App # noqa + +# from functools import reduce +# PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 +# basically, the port number is 9984 + +# The following variable is used by `bigchaindb configure` to +# prompt the user for database values. We cannot rely on +# _base_database_localmongodb.keys() because dicts are unordered. +# I tried to configure + +_database_keys_map = { + 'localmongodb': ('host', 'port', 'name'), +} + +_base_database_localmongodb = { + 'host': 'localhost', + 'port': 27017, + 'name': 'bigchain', + 'replicaset': None, + 'login': None, + 'password': None, +} + +_database_localmongodb = { + 'backend': 'localmongodb', + 'connection_timeout': 5000, + 'max_tries': 3, + 'ssl': False, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None, +} +_database_localmongodb.update(_base_database_localmongodb) + +_database_map = { + 'localmongodb': _database_localmongodb, +} + +config = { + 'server': { + # Note: this section supports all the Gunicorn settings: + # - http://docs.gunicorn.org/en/stable/settings.html + 'bind': 'localhost:9984', + 'loglevel': logging.getLevelName( + log_config['handlers']['console']['level']).lower(), + 'workers': None, # if None, the value will be cpu_count * 2 + 1 + }, + 'wsserver': { + 'scheme': 'ws', + 'host': 'localhost', + 'port': 9985, + 'advertised_scheme': 'ws', + 'advertised_host': 'localhost', + 'advertised_port': 9985, + }, + 'tendermint': { + 'host': 'localhost', + 'port': 26657, + 'version': 'v0.31.5', # look for __tm_supported_versions__ + }, + # FIXME: hardcoding to localmongodb for now + 'database': _database_map['localmongodb'], + 'log': { + 'file': log_config['handlers']['file']['filename'], + 'error_file': log_config['handlers']['errors']['filename'], + 'level_console': logging.getLevelName( + log_config['handlers']['console']['level']).lower(), + 'level_logfile': logging.getLevelName( + log_config['handlers']['file']['level']).lower(), + 'datefmt_console': log_config['formatters']['console']['datefmt'], + 'datefmt_logfile': log_config['formatters']['file']['datefmt'], + 'fmt_console': log_config['formatters']['console']['format'], + 'fmt_logfile': log_config['formatters']['file']['format'], + 'granular_levels': {}, + }, +} + +# We need to maintain a backup copy of the original config dict in case +# the user wants to reconfigure the node. Check ``bigchaindb.config_utils`` +# for more info. +_config = copy.deepcopy(config) +from bigchaindb.common.transaction import Transaction # noqa +from bigchaindb import models # noqa +from bigchaindb.upsert_validator import ValidatorElection # noqa +from bigchaindb.elections.vote import Vote # noqa + +Transaction.register_type(Transaction.CREATE, models.Transaction) +Transaction.register_type(Transaction.TRANSFER, models.Transaction) +Transaction.register_type(ValidatorElection.OPERATION, ValidatorElection) +Transaction.register_type(ChainMigrationElection.OPERATION, ChainMigrationElection) +Transaction.register_type(Vote.OPERATION, Vote) diff --git a/bigchaindb/backend/README.md b/bigchaindb/backend/README.md new file mode 100644 index 0000000..885cb9e --- /dev/null +++ b/bigchaindb/backend/README.md @@ -0,0 +1,51 @@ + + +# Backend Interfaces + +## Structure + +- [`connection.py`](./connection.py): Database connection-related interfaces +- [`query.py`](./query.py): Database query-related interfaces, dispatched through single-dispatch +- [`schema.py`](./schema.py): Database setup and schema-related interfaces, dispatched through + single-dispatch + +Built-in implementations (e.g. [MongoDB's](./localmongodb)) are provided in sub-directories and +have their connection type's location exposed as `BACKENDS` in [`connection.py`](./connection.py). + +## Single-Dispatched Interfaces + +The architecture of this module is based heavily upon Python's newly-introduced [single-dispatch +generic functions](https://www.python.org/dev/peps/pep-0443/). Single-dispatch is convenient, +because it allows Python, rather than something we design ourselves, to manage the dispatching of +generic functions based on certain conditions being met (e.g. the database backend to use). + +To see what this looks like in Planetmint, first note that our backend interfaces have been +configured to dispatch based on a backend's **connection type**. + +Call `bigchaindb.backend.connect()` to create an instance of a `Connection`: + +```python +from bigchaindb.backend import connect +connection = connect() # By default, uses the current configuration for backend, host, port, etc. +``` + +Then, we can call a backend function by directly calling its interface: + +```python +from bigchaindb.backend import query +query.write_transaction(connection, ...) +``` + +Notice that we don't need to care about which backend implementation to use or how to access it. +Code can simply call the base interface function with a `Connection` instance, and single-dispatch +will handle routing the call to the actual implementation. + +Planetmint will load and register the configured backend's implementation automatically (see +`bigchaindb.backend.connect()`), so you should always just be able to call an interface function if +you have a `Connection` instance. A few helper utilities (see [`backend/utils.py`](./utils.py)) are +also provided to make registering new backend implementations easier. diff --git a/bigchaindb/backend/__init__.py b/bigchaindb/backend/__init__.py new file mode 100644 index 0000000..edc9473 --- /dev/null +++ b/bigchaindb/backend/__init__.py @@ -0,0 +1,17 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Generic backend database interfaces expected by Planetmint. + +The interfaces in this module allow Planetmint to be agnostic about its +database backend. One can configure Planetmint to use different databases as +its data store by setting the ``database.backend`` property in the +configuration or the ``PLANETMINT_DATABASE_BACKEND`` environment variable. +""" + +# Include the backend interfaces +from bigchaindb.backend import schema, query # noqa + +from bigchaindb.backend.connection import connect # noqa diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py new file mode 100644 index 0000000..0a529f0 --- /dev/null +++ b/bigchaindb/backend/connection.py @@ -0,0 +1,170 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import logging +from importlib import import_module +from itertools import repeat + +import bigchaindb +from bigchaindb.backend.exceptions import ConnectionError +from bigchaindb.backend.utils import get_bigchaindb_config_value, get_bigchaindb_config_value_or_key_error +from bigchaindb.common.exceptions import ConfigurationError + +BACKENDS = { + 'localmongodb': 'bigchaindb.backend.localmongodb.connection.LocalMongoDBConnection', +} + +logger = logging.getLogger(__name__) + + +def connect(backend=None, host=None, port=None, name=None, max_tries=None, + connection_timeout=None, replicaset=None, ssl=None, login=None, password=None, + ca_cert=None, certfile=None, keyfile=None, keyfile_passphrase=None, + crlfile=None): + """Create a new connection to the database backend. + + All arguments default to the current configuration's values if not + given. + + Args: + backend (str): the name of the backend to use. + host (str): the host to connect to. + port (int): the port to connect to. + name (str): the name of the database to use. + replicaset (str): the name of the replica set (only relevant for + MongoDB connections). + + Returns: + An instance of :class:`~bigchaindb.backend.connection.Connection` + based on the given (or defaulted) :attr:`backend`. + + Raises: + :exc:`~ConnectionError`: If the connection to the database fails. + :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend` + is not supported or could not be loaded. + :exc:`~AuthenticationError`: If there is a OperationFailure due to + Authentication failure after connecting to the database. + """ + + backend = backend or get_bigchaindb_config_value_or_key_error('backend') + host = host or get_bigchaindb_config_value_or_key_error('host') + port = port or get_bigchaindb_config_value_or_key_error('port') + dbname = name or get_bigchaindb_config_value_or_key_error('name') + # Not sure how to handle this here. This setting is only relevant for + # mongodb. + # I added **kwargs for both RethinkDBConnection and MongoDBConnection + # to handle these these additional args. In case of RethinkDBConnection + # it just does not do anything with it. + # + # UPD: RethinkDBConnection is not here anymore cause we no longer support RethinkDB. + # The problem described above might be reconsidered next time we introduce a backend, + # if it ever happens. + replicaset = replicaset or get_bigchaindb_config_value('replicaset') + ssl = ssl if ssl is not None else get_bigchaindb_config_value('ssl', False) + login = login or get_bigchaindb_config_value('login') + password = password or get_bigchaindb_config_value('password') + ca_cert = ca_cert or get_bigchaindb_config_value('ca_cert') + certfile = certfile or get_bigchaindb_config_value('certfile') + keyfile = keyfile or get_bigchaindb_config_value('keyfile') + keyfile_passphrase = keyfile_passphrase or get_bigchaindb_config_value('keyfile_passphrase', None) + crlfile = crlfile or get_bigchaindb_config_value('crlfile') + + try: + module_name, _, class_name = BACKENDS[backend].rpartition('.') + Class = getattr(import_module(module_name), class_name) + except KeyError: + raise ConfigurationError('Backend `{}` is not supported. ' + 'Planetmint currently supports {}'.format(backend, BACKENDS.keys())) + except (ImportError, AttributeError) as exc: + raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc + + logger.debug('Connection: {}'.format(Class)) + return Class(host=host, port=port, dbname=dbname, + max_tries=max_tries, connection_timeout=connection_timeout, + replicaset=replicaset, ssl=ssl, login=login, password=password, + ca_cert=ca_cert, certfile=certfile, keyfile=keyfile, + keyfile_passphrase=keyfile_passphrase, crlfile=crlfile) + + +class Connection: + """Connection class interface. + + All backend implementations should provide a connection class that inherits + from and implements this class. + """ + + def __init__(self, host=None, port=None, dbname=None, + connection_timeout=None, max_tries=None, + **kwargs): + """Create a new :class:`~.Connection` instance. + + Args: + host (str): the host to connect to. + port (int): the port to connect to. + dbname (str): the name of the database to use. + connection_timeout (int, optional): the milliseconds to wait + until timing out the database connection attempt. + Defaults to 5000ms. + max_tries (int, optional): how many tries before giving up, + if 0 then try forever. Defaults to 3. + **kwargs: arbitrary keyword arguments provided by the + configuration's ``database`` settings + """ + + dbconf = bigchaindb.config['database'] + + self.host = host or dbconf['host'] + self.port = port or dbconf['port'] + self.dbname = dbname or dbconf['name'] + self.connection_timeout = connection_timeout if connection_timeout is not None \ + else dbconf['connection_timeout'] + self.max_tries = max_tries if max_tries is not None else dbconf['max_tries'] + self.max_tries_counter = range(self.max_tries) if self.max_tries != 0 else repeat(0) + self._conn = None + + @property + def conn(self): + if self._conn is None: + self.connect() + return self._conn + + def run(self, query): + """Run a query. + + Args: + query: the query to run + Raises: + :exc:`~DuplicateKeyError`: If the query fails because of a + duplicate key constraint. + :exc:`~OperationFailure`: If the query fails for any other + reason. + :exc:`~ConnectionError`: If the connection to the database + fails. + """ + + raise NotImplementedError() + + def connect(self): + """Try to connect to the database. + + Raises: + :exc:`~ConnectionError`: If the connection to the database + fails. + """ + + attempt = 0 + for i in self.max_tries_counter: + attempt += 1 + try: + self._conn = self._connect() + except ConnectionError as exc: + logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.', + attempt, self.max_tries if self.max_tries != 0 else '∞', + self.host, self.port, self.connection_timeout) + if attempt == self.max_tries: + logger.critical('Cannot connect to the Database. Giving up.') + raise ConnectionError() from exc + else: + break diff --git a/bigchaindb/backend/exceptions.py b/bigchaindb/backend/exceptions.py new file mode 100644 index 0000000..d524597 --- /dev/null +++ b/bigchaindb/backend/exceptions.py @@ -0,0 +1,22 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from bigchaindb.exceptions import BigchainDBError + + +class BackendError(BigchainDBError): + """Top level exception for any backend exception.""" + + +class ConnectionError(BackendError): + """Exception raised when the connection to the backend fails.""" + + +class OperationError(BackendError): + """Exception raised when a backend operation fails.""" + + +class DuplicateKeyError(OperationError): + """Exception raised when an insert fails because the key is not unique""" diff --git a/bigchaindb/backend/localmongodb/__init__.py b/bigchaindb/backend/localmongodb/__init__.py new file mode 100644 index 0000000..4e44f18 --- /dev/null +++ b/bigchaindb/backend/localmongodb/__init__.py @@ -0,0 +1,28 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""MongoDB backend implementation. + +Contains a MongoDB-specific implementation of the +:mod:`~bigchaindb.backend.schema` and :mod:`~bigchaindb.backend.query` interfaces. + +You can specify Planetmint to use MongoDB as its database backend by either +setting ``database.backend`` to ``'localmongodb'`` in your configuration file, or +setting the ``PLANETMINT_DATABASE_BACKEND`` environment variable to +``'localmongodb'``. + +MongoDB is the default database backend for Planetmint. + +If configured to use MongoDB, Planetmint will automatically return instances +of :class:`~bigchaindb.backend.localmongodb.LocalMongoDBConnection` for +:func:`~bigchaindb.backend.connection.connect` and dispatch calls of the +generic backend interfaces to the implementations in this module. +""" + +# Register the single dispatched modules on import. +from bigchaindb.backend.localmongodb import schema, query # noqa + +# MongoDBConnection should always be accessed via +# ``bigchaindb.backend.connect()``. diff --git a/bigchaindb/backend/localmongodb/connection.py b/bigchaindb/backend/localmongodb/connection.py new file mode 100644 index 0000000..6ac660b --- /dev/null +++ b/bigchaindb/backend/localmongodb/connection.py @@ -0,0 +1,136 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import logging +from ssl import CERT_REQUIRED + +import pymongo + +from bigchaindb.backend.connection import Connection +from bigchaindb.backend.exceptions import (DuplicateKeyError, + OperationError, + ConnectionError) +from bigchaindb.backend.utils import get_bigchaindb_config_value +from bigchaindb.common.exceptions import ConfigurationError +from bigchaindb.utils import Lazy + +logger = logging.getLogger(__name__) + + +class LocalMongoDBConnection(Connection): + + def __init__(self, replicaset=None, ssl=None, login=None, password=None, + ca_cert=None, certfile=None, keyfile=None, + keyfile_passphrase=None, crlfile=None, **kwargs): + """Create a new Connection instance. + + Args: + replicaset (str, optional): the name of the replica set to + connect to. + **kwargs: arbitrary keyword arguments provided by the + configuration's ``database`` settings + """ + + super().__init__(**kwargs) + self.replicaset = replicaset or get_bigchaindb_config_value('replicaset') + self.ssl = ssl if ssl is not None else get_bigchaindb_config_value('ssl', False) + self.login = login or get_bigchaindb_config_value('login') + self.password = password or get_bigchaindb_config_value('password') + self.ca_cert = ca_cert or get_bigchaindb_config_value('ca_cert') + self.certfile = certfile or get_bigchaindb_config_value('certfile') + self.keyfile = keyfile or get_bigchaindb_config_value('keyfile') + self.keyfile_passphrase = keyfile_passphrase or get_bigchaindb_config_value('keyfile_passphrase') + self.crlfile = crlfile or get_bigchaindb_config_value('crlfile') + + @property + def db(self): + return self.conn[self.dbname] + + def query(self): + return Lazy() + + def collection(self, name): + """Return a lazy object that can be used to compose a query. + + Args: + name (str): the name of the collection to query. + """ + return self.query()[self.dbname][name] + + def run(self, query): + try: + try: + return query.run(self.conn) + except pymongo.errors.AutoReconnect: + logger.warning('Lost connection to the database, ' + 'retrying query.') + return query.run(self.conn) + except pymongo.errors.AutoReconnect as exc: + raise ConnectionError from exc + except pymongo.errors.DuplicateKeyError as exc: + raise DuplicateKeyError from exc + except pymongo.errors.OperationFailure as exc: + print(f'DETAILS: {exc.details}') + raise OperationError from exc + + def _connect(self): + """Try to connect to the database. + + Raises: + :exc:`~ConnectionError`: If the connection to the database + fails. + :exc:`~AuthenticationError`: If there is a OperationFailure due to + Authentication failure after connecting to the database. + :exc:`~ConfigurationError`: If there is a ConfigurationError while + connecting to the database. + """ + + try: + # FYI: the connection process might raise a + # `ServerSelectionTimeoutError`, that is a subclass of + # `ConnectionFailure`. + # The presence of ca_cert, certfile, keyfile, crlfile implies the + # use of certificates for TLS connectivity. + if self.ca_cert is None or self.certfile is None or \ + self.keyfile is None or self.crlfile is None: + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl, + **MONGO_OPTS) + if self.login is not None and self.password is not None: + client[self.dbname].authenticate(self.login, self.password) + else: + logger.info('Connecting to MongoDB over TLS/SSL...') + client = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout, + ssl=self.ssl, + ssl_ca_certs=self.ca_cert, + ssl_certfile=self.certfile, + ssl_keyfile=self.keyfile, + ssl_pem_passphrase=self.keyfile_passphrase, + ssl_crlfile=self.crlfile, + ssl_cert_reqs=CERT_REQUIRED, + **MONGO_OPTS) + if self.login is not None: + client[self.dbname].authenticate(self.login, + mechanism='MONGODB-X509') + + return client + + except (pymongo.errors.ConnectionFailure, + pymongo.errors.OperationFailure) as exc: + logger.info('Exception in _connect(): {}'.format(exc)) + raise ConnectionError(str(exc)) from exc + except pymongo.errors.ConfigurationError as exc: + raise ConfigurationError from exc + + +MONGO_OPTS = { + 'socketTimeoutMS': 20000, +} diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py new file mode 100644 index 0000000..b63f038 --- /dev/null +++ b/bigchaindb/backend/localmongodb/query.py @@ -0,0 +1,377 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Query implementation for MongoDB""" + +from pymongo import DESCENDING + +from bigchaindb import backend +from bigchaindb.backend.exceptions import DuplicateKeyError +from bigchaindb.backend.utils import module_dispatch_registrar +from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection +from bigchaindb.common.transaction import Transaction + +register_query = module_dispatch_registrar(backend.query) + + +@register_query(LocalMongoDBConnection) +def store_transactions(conn, signed_transactions): + return conn.run(conn.collection('transactions') + .insert_many(signed_transactions)) + + +@register_query(LocalMongoDBConnection) +def get_transaction(conn, transaction_id): + return conn.run( + conn.collection('transactions') + .find_one({'id': transaction_id}, {'_id': 0})) + + +@register_query(LocalMongoDBConnection) +def get_transactions(conn, transaction_ids): + try: + return conn.run( + conn.collection('transactions') + .find({'id': {'$in': transaction_ids}}, + projection={'_id': False})) + except IndexError: + pass + + +@register_query(LocalMongoDBConnection) +def store_metadatas(conn, metadata): + return conn.run( + conn.collection('metadata') + .insert_many(metadata, ordered=False)) + + +@register_query(LocalMongoDBConnection) +def get_metadata(conn, transaction_ids): + return conn.run( + conn.collection('metadata') + .find({'id': {'$in': transaction_ids}}, + projection={'_id': False})) + + +@register_query(LocalMongoDBConnection) +def store_asset(conn, asset): + try: + return conn.run( + conn.collection('assets') + .insert_one(asset)) + except DuplicateKeyError: + pass + + +@register_query(LocalMongoDBConnection) +def store_assets(conn, assets): + return conn.run( + conn.collection('assets') + .insert_many(assets, ordered=False)) + + +@register_query(LocalMongoDBConnection) +def get_asset(conn, asset_id): + try: + return conn.run( + conn.collection('assets') + .find_one({'id': asset_id}, {'_id': 0, 'id': 0})) + except IndexError: + pass + + +@register_query(LocalMongoDBConnection) +def get_assets(conn, asset_ids): + return conn.run( + conn.collection('assets') + .find({'id': {'$in': asset_ids}}, + projection={'_id': False})) + + +@register_query(LocalMongoDBConnection) +def get_spent(conn, transaction_id, output): + query = {'inputs': + {'$elemMatch': + {'$and': [{'fulfills.transaction_id': transaction_id}, + {'fulfills.output_index': output}]}}} + + return conn.run( + conn.collection('transactions') + .find(query, {'_id': 0})) + + +@register_query(LocalMongoDBConnection) +def get_latest_block(conn): + return conn.run( + conn.collection('blocks') + .find_one(projection={'_id': False}, + sort=[('height', DESCENDING)])) + + +@register_query(LocalMongoDBConnection) +def store_block(conn, block): + try: + return conn.run( + conn.collection('blocks') + .insert_one(block)) + except DuplicateKeyError: + pass + + +@register_query(LocalMongoDBConnection) +def get_txids_filtered(conn, asset_id, operation=None, last_tx=None): + + match = { + Transaction.CREATE: {'operation': 'CREATE', 'id': asset_id}, + Transaction.TRANSFER: {'operation': 'TRANSFER', 'asset.id': asset_id}, + None: {'$or': [{'asset.id': asset_id}, {'id': asset_id}]}, + }[operation] + + cursor = conn.run(conn.collection('transactions').find(match)) + + if last_tx: + cursor = cursor.sort([('$natural', DESCENDING)]).limit(1) + + return (elem['id'] for elem in cursor) + + +@register_query(LocalMongoDBConnection) +def text_search(conn, search, *, language='english', case_sensitive=False, + diacritic_sensitive=False, text_score=False, limit=0, table='assets'): + cursor = conn.run( + conn.collection(table) + .find({'$text': { + '$search': search, + '$language': language, + '$caseSensitive': case_sensitive, + '$diacriticSensitive': diacritic_sensitive}}, + {'score': {'$meta': 'textScore'}, '_id': False}) + .sort([('score', {'$meta': 'textScore'})]) + .limit(limit)) + + if text_score: + return cursor + + return (_remove_text_score(obj) for obj in cursor) + + +def _remove_text_score(asset): + asset.pop('score', None) + return asset + + +@register_query(LocalMongoDBConnection) +def get_owned_ids(conn, owner): + cursor = conn.run( + conn.collection('transactions').aggregate([ + {'$match': {'outputs.public_keys': owner}}, + {'$project': {'_id': False}} + ])) + return cursor + + +@register_query(LocalMongoDBConnection) +def get_spending_transactions(conn, inputs): + transaction_ids = [i['transaction_id'] for i in inputs] + output_indexes = [i['output_index'] for i in inputs] + query = {'inputs': + {'$elemMatch': + {'$and': + [ + {'fulfills.transaction_id': {'$in': transaction_ids}}, + {'fulfills.output_index': {'$in': output_indexes}} + ]}}} + + cursor = conn.run( + conn.collection('transactions').find(query, {'_id': False})) + return cursor + + +@register_query(LocalMongoDBConnection) +def get_block(conn, block_id): + return conn.run( + conn.collection('blocks') + .find_one({'height': block_id}, + projection={'_id': False})) + + +@register_query(LocalMongoDBConnection) +def get_block_with_transaction(conn, txid): + return conn.run( + conn.collection('blocks') + .find({'transactions': txid}, + projection={'_id': False, 'height': True})) + + +@register_query(LocalMongoDBConnection) +def delete_transactions(conn, txn_ids): + conn.run(conn.collection('assets').delete_many({'id': {'$in': txn_ids}})) + conn.run(conn.collection('metadata').delete_many({'id': {'$in': txn_ids}})) + conn.run(conn.collection('transactions').delete_many({'id': {'$in': txn_ids}})) + + +@register_query(LocalMongoDBConnection) +def store_unspent_outputs(conn, *unspent_outputs): + if unspent_outputs: + try: + return conn.run( + conn.collection('utxos').insert_many( + unspent_outputs, + ordered=False, + ) + ) + except DuplicateKeyError: + # TODO log warning at least + pass + + +@register_query(LocalMongoDBConnection) +def delete_unspent_outputs(conn, *unspent_outputs): + if unspent_outputs: + return conn.run( + conn.collection('utxos').delete_many({ + '$or': [{ + '$and': [ + {'transaction_id': unspent_output['transaction_id']}, + {'output_index': unspent_output['output_index']}, + ], + } for unspent_output in unspent_outputs] + }) + ) + + +@register_query(LocalMongoDBConnection) +def get_unspent_outputs(conn, *, query=None): + if query is None: + query = {} + return conn.run(conn.collection('utxos').find(query, + projection={'_id': False})) + + +@register_query(LocalMongoDBConnection) +def store_pre_commit_state(conn, state): + return conn.run( + conn.collection('pre_commit') + .replace_one({}, state, upsert=True) + ) + + +@register_query(LocalMongoDBConnection) +def get_pre_commit_state(conn): + return conn.run(conn.collection('pre_commit').find_one()) + + +@register_query(LocalMongoDBConnection) +def store_validator_set(conn, validators_update): + height = validators_update['height'] + return conn.run( + conn.collection('validators').replace_one( + {'height': height}, + validators_update, + upsert=True + ) + ) + + +@register_query(LocalMongoDBConnection) +def delete_validator_set(conn, height): + return conn.run( + conn.collection('validators').delete_many({'height': height}) + ) + + +@register_query(LocalMongoDBConnection) +def store_election(conn, election_id, height, is_concluded): + return conn.run( + conn.collection('elections').replace_one( + {'election_id': election_id, + 'height': height}, + {'election_id': election_id, + 'height': height, + 'is_concluded': is_concluded}, + upsert=True, + ) + ) + + +@register_query(LocalMongoDBConnection) +def store_elections(conn, elections): + return conn.run( + conn.collection('elections').insert_many(elections) + ) + + +@register_query(LocalMongoDBConnection) +def delete_elections(conn, height): + return conn.run( + conn.collection('elections').delete_many({'height': height}) + ) + + +@register_query(LocalMongoDBConnection) +def get_validator_set(conn, height=None): + query = {} + if height is not None: + query = {'height': {'$lte': height}} + + cursor = conn.run( + conn.collection('validators') + .find(query, projection={'_id': False}) + .sort([('height', DESCENDING)]) + .limit(1) + ) + + return next(cursor, None) + + +@register_query(LocalMongoDBConnection) +def get_election(conn, election_id): + query = {'election_id': election_id} + + return conn.run( + conn.collection('elections') + .find_one(query, projection={'_id': False}, + sort=[('height', DESCENDING)]) + ) + + +@register_query(LocalMongoDBConnection) +def get_asset_tokens_for_public_key(conn, asset_id, public_key): + query = {'outputs.public_keys': [public_key], + 'asset.id': asset_id} + + cursor = conn.run( + conn.collection('transactions').aggregate([ + {'$match': query}, + {'$project': {'_id': False}} + ])) + return cursor + + +@register_query(LocalMongoDBConnection) +def store_abci_chain(conn, height, chain_id, is_synced=True): + return conn.run( + conn.collection('abci_chains').replace_one( + {'height': height}, + {'height': height, 'chain_id': chain_id, + 'is_synced': is_synced}, + upsert=True, + ) + ) + + +@register_query(LocalMongoDBConnection) +def delete_abci_chain(conn, height): + return conn.run( + conn.collection('abci_chains').delete_many({'height': height}) + ) + + +@register_query(LocalMongoDBConnection) +def get_latest_abci_chain(conn): + return conn.run( + conn.collection('abci_chains') + .find_one(projection={'_id': False}, sort=[('height', DESCENDING)]) + ) diff --git a/bigchaindb/backend/localmongodb/schema.py b/bigchaindb/backend/localmongodb/schema.py new file mode 100644 index 0000000..d520521 --- /dev/null +++ b/bigchaindb/backend/localmongodb/schema.py @@ -0,0 +1,90 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Utils to initialize and drop the database.""" + +import logging + +from pymongo import ASCENDING, DESCENDING, TEXT +from pymongo.errors import CollectionInvalid + +from bigchaindb import backend +from bigchaindb.backend.utils import module_dispatch_registrar +from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection + + +logger = logging.getLogger(__name__) +register_schema = module_dispatch_registrar(backend.schema) + + +INDEXES = { + 'transactions': [ + ('id', dict(unique=True, name='transaction_id')), + ('asset.id', dict(name='asset_id')), + ('outputs.public_keys', dict(name='outputs')), + ([('inputs.fulfills.transaction_id', ASCENDING), + ('inputs.fulfills.output_index', ASCENDING)], dict(name='inputs')), + ], + 'assets': [ + ('id', dict(name='asset_id', unique=True)), + ([('$**', TEXT)], dict(name='text')), + ], + 'blocks': [ + ([('height', DESCENDING)], dict(name='height', unique=True)), + ], + 'metadata': [ + ('id', dict(name='transaction_id', unique=True)), + ([('$**', TEXT)], dict(name='text')), + ], + 'utxos': [ + ([('transaction_id', ASCENDING), + ('output_index', ASCENDING)], dict(name='utxo', unique=True)), + ], + 'pre_commit': [ + ('height', dict(name='height', unique=True)), + ], + 'elections': [ + ([('height', DESCENDING), ('election_id', ASCENDING)], + dict(name='election_id_height', unique=True)), + ], + 'validators': [ + ('height', dict(name='height', unique=True)), + ], + 'abci_chains': [ + ('height', dict(name='height', unique=True)), + ('chain_id', dict(name='chain_id', unique=True)), + ], +} + + +@register_schema(LocalMongoDBConnection) +def create_database(conn, dbname): + logger.info('Create database `%s`.', dbname) + # TODO: read and write concerns can be declared here + conn.conn.get_database(dbname) + + +@register_schema(LocalMongoDBConnection) +def create_tables(conn, dbname): + for table_name in backend.schema.TABLES: + # create the table + # TODO: read and write concerns can be declared here + try: + logger.info(f'Create `{table_name}` table.') + conn.conn[dbname].create_collection(table_name) + except CollectionInvalid: + logger.info(f'Collection {table_name} already exists.') + create_indexes(conn, dbname, table_name, INDEXES[table_name]) + + +def create_indexes(conn, dbname, collection, indexes): + logger.info(f'Ensure secondary indexes for `{collection}`.') + for fields, kwargs in indexes: + conn.conn[dbname][collection].create_index(fields, **kwargs) + + +@register_schema(LocalMongoDBConnection) +def drop_database(conn, dbname): + conn.conn.drop_database(dbname) diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py new file mode 100644 index 0000000..184f99f --- /dev/null +++ b/bigchaindb/backend/query.py @@ -0,0 +1,430 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Query interfaces for backends.""" + +from functools import singledispatch + +from bigchaindb.backend.exceptions import OperationError + + +@singledispatch +def store_asset(connection, asset): + """Write an asset to the asset table. + + Args: + asset (dict): the asset. + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def store_assets(connection, assets): + """Write a list of assets to the assets table. + + Args: + assets (list): a list of assets to write. + + Returns: + The database response. + """ + + raise NotImplementedError + + +@singledispatch +def store_metadatas(connection, metadata): + """Write a list of metadata to metadata table. + + Args: + metadata (list): list of metadata. + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def store_transactions(connection, signed_transactions): + """Store the list of transactions.""" + + raise NotImplementedError + + +@singledispatch +def get_transaction(connection, transaction_id): + """Get a transaction from the transactions table. + + Args: + transaction_id (str): the id of the transaction. + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def get_transactions(connection, transaction_ids): + """Get transactions from the transactions table. + + Args: + transaction_ids (list): list of transaction ids to fetch + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def get_asset(connection, asset_id): + """Get a transaction from the transactions table. + + Args: + asset_id (str): the id of the asset + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def get_spent(connection, transaction_id, condition_id): + """Check if a `txid` was already used as an input. + + A transaction can be used as an input for another transaction. Bigchain + needs to make sure that a given `txid` is only used once. + + Args: + transaction_id (str): The id of the transaction. + condition_id (int): The index of the condition in the respective + transaction. + + Returns: + The transaction that used the `txid` as an input else `None` + """ + + raise NotImplementedError + + +@singledispatch +def get_spending_transactions(connection, inputs): + """Return transactions which spend given inputs + + Args: + inputs (list): list of {txid, output} + + Returns: + Iterator of (block_ids, transaction) for transactions that + spend given inputs. + """ + raise NotImplementedError + + +@singledispatch +def get_owned_ids(connection, owner): + """Retrieve a list of `txids` that can we used has inputs. + + Args: + owner (str): base58 encoded public key. + + Returns: + Iterator of (block_id, transaction) for transactions + that list given owner in conditions. + """ + raise NotImplementedError + + +@singledispatch +def get_block(connection, block_id): + """Get a block from the bigchain table. + + Args: + block_id (str): block id of the block to get + + Returns: + block (dict): the block or `None` + """ + + raise NotImplementedError + + +@singledispatch +def get_block_with_transaction(connection, txid): + """Get a block containing transaction id `txid` + + Args: + txid (str): id of transaction to be searched. + + Returns: + block_id (int): the block id or `None` + """ + + raise NotImplementedError + + +@singledispatch +def get_metadata(connection, transaction_ids): + """Get a list of metadata from the metadata table. + + Args: + transaction_ids (list): a list of ids for the metadata to be retrieved from + the database. + + Returns: + metadata (list): the list of returned metadata. + """ + raise NotImplementedError + + +@singledispatch +def get_assets(connection, asset_ids): + """Get a list of assets from the assets table. + Args: + asset_ids (list): a list of ids for the assets to be retrieved from + the database. + Returns: + assets (list): the list of returned assets. + """ + raise NotImplementedError + + +@singledispatch +def get_txids_filtered(connection, asset_id, operation=None): + """Return all transactions for a particular asset id and optional operation. + + Args: + asset_id (str): ID of transaction that defined the asset + operation (str) (optional): Operation to filter on + """ + + raise NotImplementedError + + +@singledispatch +def text_search(conn, search, *, language='english', case_sensitive=False, + diacritic_sensitive=False, text_score=False, limit=0, table=None): + """Return all the assets that match the text search. + + The results are sorted by text score. + For more information about the behavior of text search on MongoDB see + https://docs.mongodb.com/manual/reference/operator/query/text/#behavior + + Args: + search (str): Text search string to query the text index + language (str, optional): The language for the search and the rules for + stemmer and tokenizer. If the language is ``None`` text search uses + simple tokenization and no stemming. + case_sensitive (bool, optional): Enable or disable case sensitive + search. + diacritic_sensitive (bool, optional): Enable or disable case sensitive + diacritic search. + text_score (bool, optional): If ``True`` returns the text score with + each document. + limit (int, optional): Limit the number of returned documents. + + Returns: + :obj:`list` of :obj:`dict`: a list of assets + + Raises: + OperationError: If the backend does not support text search + """ + + raise OperationError('This query is only supported when running ' + 'Planetmint with MongoDB as the backend.') + + +@singledispatch +def get_latest_block(conn): + """Get the latest commited block i.e. block with largest height""" + + raise NotImplementedError + + +@singledispatch +def store_block(conn, block): + """Write a new block to the `blocks` table + + Args: + block (dict): block with current height and block hash. + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def store_unspent_outputs(connection, unspent_outputs): + """Store unspent outputs in ``utxo_set`` table.""" + + raise NotImplementedError + + +@singledispatch +def delete_unspent_outputs(connection, unspent_outputs): + """Delete unspent outputs in ``utxo_set`` table.""" + + raise NotImplementedError + + +@singledispatch +def delete_transactions(conn, txn_ids): + """Delete transactions from database + + Args: + txn_ids (list): list of transaction ids + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def get_unspent_outputs(connection, *, query=None): + """Retrieves unspent outputs. + + Args: + query (dict): An optional parameter to filter the result set. + Defaults to ``None``, which means that all UTXO records + will be returned. + + Returns: + Generator yielding unspent outputs (UTXO set) according to the + given query. + """ + + raise NotImplementedError + + +@singledispatch +def store_pre_commit_state(connection, state): + """Store pre-commit state. + + Args: + state (dict): pre-commit state. + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + +@singledispatch +def get_pre_commit_state(connection): + """Get pre-commit state. + + Returns: + Document representing the pre-commit state. + """ + + raise NotImplementedError + + +@singledispatch +def store_validator_set(conn, validator_update): + """Store updated validator set""" + + raise NotImplementedError + + +@singledispatch +def delete_validator_set(conn, height): + """Delete the validator set at the given height.""" + + raise NotImplementedError + + +@singledispatch +def store_election(conn, election_id, height, is_concluded): + """Store election record""" + + raise NotImplementedError + + +@singledispatch +def store_elections(conn, elections): + """Store election records in bulk""" + + raise NotImplementedError + + +@singledispatch +def delete_elections(conn, height): + """Delete all election records at the given height""" + + raise NotImplementedError + + +@singledispatch +def get_validator_set(conn, height): + """Get validator set for a given `height`, if `height` is not specified + then return the latest validator set + """ + + raise NotImplementedError + + +@singledispatch +def get_election(conn, election_id): + """Return the election record + """ + + raise NotImplementedError + + +@singledispatch +def get_asset_tokens_for_public_key(connection, asset_id, public_key): + """Retrieve a list of tokens of type `asset_id` that are owned by the `public_key`. + Args: + asset_id (str): Id of the token. + public_key (str): base58 encoded public key + Returns: + Iterator of transaction that list given owner in conditions. + """ + raise NotImplementedError + + +@singledispatch +def store_abci_chain(conn, height, chain_id, is_synced=True): + """Create or update an ABCI chain at the given height. + Usually invoked in the beginning of the ABCI communications (height=0) + or when ABCI client (like Tendermint) is migrated (any height). + + Args: + is_synced: True if the chain is known by both ABCI client and server + """ + + raise NotImplementedError + + +@singledispatch +def delete_abci_chain(conn, height): + """Delete the ABCI chain at the given height.""" + + raise NotImplementedError + + +@singledispatch +def get_latest_abci_chain(conn): + """Returns the ABCI chain stored at the biggest height, if any, + None otherwise. + """ + raise NotImplementedError diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py new file mode 100644 index 0000000..3856b1b --- /dev/null +++ b/bigchaindb/backend/schema.py @@ -0,0 +1,128 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Database creation and schema-providing interfaces for backends.""" + +from functools import singledispatch +import logging + +import bigchaindb +from bigchaindb.backend.connection import connect +from bigchaindb.common.exceptions import ValidationError +from bigchaindb.common.utils import validate_all_values_for_key_in_obj, validate_all_values_for_key_in_list + +logger = logging.getLogger(__name__) + +# Tables/collections that every backend database must create +TABLES = ('transactions', 'blocks', 'assets', 'metadata', + 'validators', 'elections', 'pre_commit', 'utxos', 'abci_chains') + +VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', + 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', + 'russian', 'spanish', 'swedish', 'turkish', 'none', + 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', + 'ro', 'ru', 'es', 'sv', 'tr') + + +@singledispatch +def create_database(connection, dbname): + """Create database to be used by Planetmint. + + Args: + dbname (str): the name of the database to create. + """ + + raise NotImplementedError + + +@singledispatch +def create_tables(connection, dbname): + """Create the tables to be used by Planetmint. + + Args: + dbname (str): the name of the database to create tables for. + """ + + raise NotImplementedError + + +@singledispatch +def drop_database(connection, dbname): + """Drop the database used by Planetmint. + + Args: + dbname (str): the name of the database to drop. + + Raises: + :exc:`~DatabaseDoesNotExist`: If the given :attr:`dbname` does not + exist as a database. + """ + + raise NotImplementedError + + +def init_database(connection=None, dbname=None): + """Initialize the configured backend for use with Planetmint. + + Creates a database with :attr:`dbname` with any required tables + and supporting indexes. + + Args: + connection (:class:`~bigchaindb.backend.connection.Connection`): an + existing connection to use to initialize the database. + Creates one if not given. + dbname (str): the name of the database to create. + Defaults to the database name given in the Planetmint + configuration. + """ + + connection = connection or connect() + dbname = dbname or bigchaindb.config['database']['name'] + + create_database(connection, dbname) + create_tables(connection, dbname) + + +def validate_language_key(obj, key): + """Validate all nested "language" key in `obj`. + + Args: + obj (dict): dictionary whose "language" key is to be validated. + + Returns: + None: validation successful + + Raises: + ValidationError: will raise exception in case language is not valid. + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'localmongodb': + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_values_for_key_in_obj(data, 'language', validate_language) + elif isinstance(data, list): + validate_all_values_for_key_in_list(data, 'language', validate_language) + + +def validate_language(value): + """Check if `value` is a valid language. + https://docs.mongodb.com/manual/reference/text-search-languages/ + + Args: + value (str): language to validated + + Returns: + None: validation successful + + Raises: + ValidationError: will raise exception in case language is not valid. + """ + if value not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(value) + raise ValidationError(error_str) diff --git a/bigchaindb/backend/utils.py b/bigchaindb/backend/utils.py new file mode 100644 index 0000000..9bca8ac --- /dev/null +++ b/bigchaindb/backend/utils.py @@ -0,0 +1,39 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import bigchaindb + + +class ModuleDispatchRegistrationError(Exception): + """Raised when there is a problem registering dispatched functions for a + module + """ + + +def module_dispatch_registrar(module): + def dispatch_wrapper(obj_type): + def wrapper(func): + func_name = func.__name__ + try: + dispatch_registrar = getattr(module, func_name) + return dispatch_registrar.register(obj_type)(func) + except AttributeError as ex: + raise ModuleDispatchRegistrationError( + ('`{module}` does not contain a single-dispatchable ' + 'function named `{func}`. The module being registered ' + 'was not implemented correctly!').format( + func=func_name, module=module.__name__)) from ex + + return wrapper + + return dispatch_wrapper + + +def get_bigchaindb_config_value(key, default_value=None): + return bigchaindb.config['database'].get(key, default_value) + + +def get_bigchaindb_config_value_or_key_error(key): + return bigchaindb.config['database'][key] diff --git a/bigchaindb/commands/__init__.py b/bigchaindb/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py new file mode 100644 index 0000000..8801986 --- /dev/null +++ b/bigchaindb/commands/bigchaindb.py @@ -0,0 +1,398 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Implementation of the `bigchaindb` command, +the command-line interface (CLI) for Planetmint Server. +""" + +import os +import logging +import argparse +import copy +import json +import sys + +from bigchaindb.core import rollback +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection +from bigchaindb.utils import load_node_key +from bigchaindb.common.transaction_mode_types import BROADCAST_TX_COMMIT +from bigchaindb.common.exceptions import (DatabaseDoesNotExist, + ValidationError) +from bigchaindb.elections.vote import Vote +import bigchaindb +from bigchaindb import (backend, ValidatorElection, + BigchainDB) +from bigchaindb.backend import schema +from bigchaindb.commands import utils +from bigchaindb.commands.utils import (configure_bigchaindb, + input_on_stderr) +from bigchaindb.log import setup_logging +from bigchaindb.tendermint_utils import public_key_from_base64 +from bigchaindb.commands.election_types import elections +from bigchaindb.version import __tm_supported_versions__ + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +# Note about printing: +# We try to print to stdout for results of a command that may be useful to +# someone (or another program). Strictly informational text, or errors, +# should be printed to stderr. + + +@configure_bigchaindb +def run_show_config(args): + """Show the current configuration""" + # TODO Proposal: remove the "hidden" configuration. Only show config. If + # the system needs to be configured, then display information on how to + # configure the system. + config = copy.deepcopy(bigchaindb.config) + del config['CONFIGURED'] + print(json.dumps(config, indent=4, sort_keys=True)) + + +@configure_bigchaindb +def run_configure(args): + """Run a script to configure the current node.""" + config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH + + config_file_exists = False + # if the config path is `-` then it's stdout + if config_path != '-': + config_file_exists = os.path.exists(config_path) + + if config_file_exists and not args.yes: + want = input_on_stderr('Config file `{}` exists, do you want to ' + 'override it? (cannot be undone) [y/N]: '.format(config_path)) + if want != 'y': + return + + conf = copy.deepcopy(bigchaindb.config) + + # select the correct config defaults based on the backend + print('Generating default configuration for backend {}' + .format(args.backend), file=sys.stderr) + database_keys = bigchaindb._database_keys_map[args.backend] + conf['database'] = bigchaindb._database_map[args.backend] + + if not args.yes: + for key in ('bind', ): + val = conf['server'][key] + conf['server'][key] = input_on_stderr('API Server {}? (default `{}`): '.format(key, val), val) + + for key in ('scheme', 'host', 'port'): + val = conf['wsserver'][key] + conf['wsserver'][key] = input_on_stderr('WebSocket Server {}? (default `{}`): '.format(key, val), val) + + for key in database_keys: + val = conf['database'][key] + conf['database'][key] = input_on_stderr('Database {}? (default `{}`): '.format(key, val), val) + + for key in ('host', 'port'): + val = conf['tendermint'][key] + conf['tendermint'][key] = input_on_stderr('Tendermint {}? (default `{}`)'.format(key, val), val) + + if config_path != '-': + bigchaindb.config_utils.write_config(conf, config_path) + else: + print(json.dumps(conf, indent=4, sort_keys=True)) + print('Configuration written to {}'.format(config_path), file=sys.stderr) + print('Ready to go!', file=sys.stderr) + + +@configure_bigchaindb +def run_election(args): + """Initiate and manage elections""" + + b = BigchainDB() + + # Call the function specified by args.action, as defined above + globals()[f'run_election_{args.action}'](args, b) + + +def run_election_new(args, bigchain): + election_type = args.election_type.replace('-', '_') + globals()[f'run_election_new_{election_type}'](args, bigchain) + + +def create_new_election(sk, bigchain, election_class, data): + try: + key = load_node_key(sk) + voters = election_class.recipients(bigchain) + election = election_class.generate([key.public_key], + voters, + data, None).sign([key.private_key]) + election.validate(bigchain) + except ValidationError as e: + logger.error(e) + return False + except FileNotFoundError as fd_404: + logger.error(fd_404) + return False + + resp = bigchain.write_transaction(election, BROADCAST_TX_COMMIT) + if resp == (202, ''): + logger.info('[SUCCESS] Submitted proposal with id: {}'.format(election.id)) + return election.id + else: + logger.error('Failed to commit election proposal') + return False + + +def run_election_new_upsert_validator(args, bigchain): + """Initiates an election to add/update/remove a validator to an existing Planetmint network + + :param args: dict + args = { + 'public_key': the public key of the proposed peer, (str) + 'power': the proposed validator power for the new peer, (str) + 'node_id': the node_id of the new peer (str) + 'sk': the path to the private key of the node calling the election (str) + } + :param bigchain: an instance of Planetmint + :return: election_id or `False` in case of failure + """ + + new_validator = { + 'public_key': {'value': public_key_from_base64(args.public_key), + 'type': 'ed25519-base16'}, + 'power': args.power, + 'node_id': args.node_id + } + + return create_new_election(args.sk, bigchain, ValidatorElection, new_validator) + + +def run_election_new_chain_migration(args, bigchain): + """Initiates an election to halt block production + + :param args: dict + args = { + 'sk': the path to the private key of the node calling the election (str) + } + :param bigchain: an instance of Planetmint + :return: election_id or `False` in case of failure + """ + + return create_new_election(args.sk, bigchain, ChainMigrationElection, {}) + + +def run_election_approve(args, bigchain): + """Approve an election + + :param args: dict + args = { + 'election_id': the election_id of the election (str) + 'sk': the path to the private key of the signer (str) + } + :param bigchain: an instance of Planetmint + :return: success log message or `False` in case of error + """ + + key = load_node_key(args.sk) + tx = bigchain.get_transaction(args.election_id) + voting_powers = [v.amount for v in tx.outputs if key.public_key in v.public_keys] + if len(voting_powers) > 0: + voting_power = voting_powers[0] + else: + logger.error('The key you provided does not match any of the eligible voters in this election.') + return False + + inputs = [i for i in tx.to_inputs() if key.public_key in i.owners_before] + election_pub_key = ValidatorElection.to_public_key(tx.id) + approval = Vote.generate(inputs, + [([election_pub_key], voting_power)], + tx.id).sign([key.private_key]) + approval.validate(bigchain) + + resp = bigchain.write_transaction(approval, BROADCAST_TX_COMMIT) + + if resp == (202, ''): + logger.info('[SUCCESS] Your vote has been submitted') + return approval.id + else: + logger.error('Failed to commit vote') + return False + + +def run_election_show(args, bigchain): + """Retrieves information about an election + + :param args: dict + args = { + 'election_id': the transaction_id for an election (str) + } + :param bigchain: an instance of Planetmint + """ + + election = bigchain.get_transaction(args.election_id) + if not election: + logger.error(f'No election found with election_id {args.election_id}') + return + + response = election.show_election(bigchain) + + logger.info(response) + + return response + + +def _run_init(): + bdb = bigchaindb.BigchainDB() + + schema.init_database(connection=bdb.connection) + + +@configure_bigchaindb +def run_init(args): + """Initialize the database""" + _run_init() + + +@configure_bigchaindb +def run_drop(args): + """Drop the database""" + dbname = bigchaindb.config['database']['name'] + + if not args.yes: + response = input_on_stderr('Do you want to drop `{}` database? [y/n]: '.format(dbname)) + if response != 'y': + return + + conn = backend.connect() + try: + schema.drop_database(conn, dbname) + except DatabaseDoesNotExist: + print("Cannot drop '{name}'. The database does not exist.".format(name=dbname), file=sys.stderr) + + +def run_recover(b): + rollback(b) + + +@configure_bigchaindb +def run_start(args): + """Start the processes to run the node""" + + # Configure Logging + setup_logging() + + logger.info('Planetmint Version %s', bigchaindb.__version__) + run_recover(bigchaindb.lib.BigchainDB()) + + if not args.skip_initialize_database: + logger.info('Initializing database') + _run_init() + + logger.info('Starting Planetmint main process.') + from bigchaindb.start import start + start(args) + + +def run_tendermint_version(args): + """Show the supported Tendermint version(s)""" + supported_tm_ver = { + 'description': 'Planetmint supports the following Tendermint version(s)', + 'tendermint': __tm_supported_versions__, + } + print(json.dumps(supported_tm_ver, indent=4, sort_keys=True)) + + +def create_parser(): + parser = argparse.ArgumentParser( + description='Control your Planetmint node.', + parents=[utils.base_parser]) + + # all the commands are contained in the subparsers object, + # the command selected by the user will be stored in `args.command` + # that is used by the `main` function to select which other + # function to call. + subparsers = parser.add_subparsers(title='Commands', + dest='command') + + # parser for writing a config file + config_parser = subparsers.add_parser('configure', + help='Prepare the config file.') + + config_parser.add_argument('backend', + choices=['localmongodb'], + default='localmongodb', + const='localmongodb', + nargs='?', + help='The backend to use. It can only be ' + '"localmongodb", currently.') + + # parser for managing elections + election_parser = subparsers.add_parser('election', + help='Manage elections.') + + election_subparser = election_parser.add_subparsers(title='Action', + dest='action') + + new_election_parser = election_subparser.add_parser('new', + help='Calls a new election.') + + new_election_subparser = new_election_parser.add_subparsers(title='Election_Type', + dest='election_type') + + # Parser factory for each type of new election, so we get a bunch of commands that look like this: + # election new ... + for name, data in elections.items(): + args = data['args'] + generic_parser = new_election_subparser.add_parser(name, help=data['help']) + for arg, kwargs in args.items(): + generic_parser.add_argument(arg, **kwargs) + + approve_election_parser = election_subparser.add_parser('approve', + help='Approve the election.') + approve_election_parser.add_argument('election_id', + help='The election_id of the election.') + approve_election_parser.add_argument('--private-key', + dest='sk', + required=True, + help='Path to the private key of the election initiator.') + + show_election_parser = election_subparser.add_parser('show', + help='Provides information about an election.') + + show_election_parser.add_argument('election_id', + help='The transaction id of the election you wish to query.') + + # parsers for showing/exporting config values + subparsers.add_parser('show-config', + help='Show the current configuration') + + # parser for database-level commands + subparsers.add_parser('init', + help='Init the database') + + subparsers.add_parser('drop', + help='Drop the database') + + # parser for starting Planetmint + start_parser = subparsers.add_parser('start', + help='Start Planetmint') + + start_parser.add_argument('--no-init', + dest='skip_initialize_database', + default=False, + action='store_true', + help='Skip database initialization') + + subparsers.add_parser('tendermint-version', + help='Show the Tendermint supported versions') + + start_parser.add_argument('--experimental-parallel-validation', + dest='experimental_parallel_validation', + default=False, + action='store_true', + help='💀 EXPERIMENTAL: parallelize validation for better throughput 💀') + + return parser + + +def main(): + utils.start(create_parser(), sys.argv[1:], globals()) diff --git a/bigchaindb/commands/election_types.py b/bigchaindb/commands/election_types.py new file mode 100644 index 0000000..cfa58b9 --- /dev/null +++ b/bigchaindb/commands/election_types.py @@ -0,0 +1,31 @@ +elections = { + 'upsert-validator': { + 'help': 'Propose a change to the validator set', + 'args': { + 'public_key': { + 'help': 'Public key of the validator to be added/updated/removed.' + }, + 'power': { + 'type': int, + 'help': 'The proposed power for the validator. Setting to 0 will remove the validator.'}, + 'node_id': { + 'help': 'The node_id of the validator.' + }, + '--private-key': { + 'dest': 'sk', + 'required': True, + 'help': 'Path to the private key of the election initiator.' + } + } + }, + 'chain-migration': { + 'help': 'Call for a halt to block production to allow for a version change across breaking changes.', + 'args': { + '--private-key': { + 'dest': 'sk', + 'required': True, + 'help': 'Path to the private key of the election initiator.' + } + } + } +} diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py new file mode 100644 index 0000000..6adb56a --- /dev/null +++ b/bigchaindb/commands/utils.py @@ -0,0 +1,161 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Utility functions and basic common arguments +for ``argparse.ArgumentParser``. +""" + +import argparse +import builtins +import functools +import multiprocessing as mp +import sys + +import bigchaindb +import bigchaindb.config_utils +from bigchaindb.version import __version__ + + +def configure_bigchaindb(command): + """Decorator to be used by command line functions, such that the + configuration of bigchaindb is performed before the execution of + the command. + + Args: + command: The command to decorate. + + Returns: + The command wrapper function. + + """ + @functools.wraps(command) + def configure(args): + config_from_cmdline = None + try: + if args.log_level is not None: + config_from_cmdline = { + 'log': { + 'level_console': args.log_level, + 'level_logfile': args.log_level, + }, + 'server': {'loglevel': args.log_level}, + } + except AttributeError: + pass + bigchaindb.config_utils.autoconfigure( + filename=args.config, config=config_from_cmdline, force=True) + command(args) + + return configure + + +def _convert(value, default=None, convert=None): + def convert_bool(value): + if value.lower() in ('true', 't', 'yes', 'y'): + return True + if value.lower() in ('false', 'f', 'no', 'n'): + return False + raise ValueError('{} cannot be converted to bool'.format(value)) + + if value == '': + value = None + + if convert is None: + if default is not None: + convert = type(default) + else: + convert = str + + if convert == bool: + convert = convert_bool + + if value is None: + return default + else: + return convert(value) + + +# We need this because `input` always prints on stdout, while it should print +# to stderr. It's a very old bug, check it out here: +# - https://bugs.python.org/issue1927 +def input_on_stderr(prompt='', default=None, convert=None): + """Output a string to stderr and wait for input. + + Args: + prompt (str): the message to display. + default: the default value to return if the user + leaves the field empty + convert (callable): a callable to be used to convert + the value the user inserted. If None, the type of + ``default`` will be used. + """ + + print(prompt, end='', file=sys.stderr) + value = builtins.input() + return _convert(value, default, convert) + + +def start(parser, argv, scope): + """Utility function to execute a subcommand. + + The function will look up in the ``scope`` + if there is a function called ``run_`` + and will run it using ``parser.args`` as first positional argument. + + Args: + parser: an ArgumentParser instance. + argv: the list of command line arguments without the script name. + scope (dict): map containing (eventually) the functions to be called. + + Raises: + NotImplementedError: if ``scope`` doesn't contain a function called + ``run_``. + """ + args = parser.parse_args(argv) + + if not args.command: + parser.print_help() + raise SystemExit() + + # look up in the current scope for a function called 'run_' + # replacing all the dashes '-' with the lowercase character '_' + func = scope.get('run_' + args.command.replace('-', '_')) + + # if no command has been found, raise a `NotImplementedError` + if not func: + raise NotImplementedError('Command `{}` not yet implemented'. + format(args.command)) + + args.multiprocess = getattr(args, 'multiprocess', False) + + if args.multiprocess is False: + args.multiprocess = 1 + elif args.multiprocess is None: + args.multiprocess = mp.cpu_count() + + return func(args) + + +base_parser = argparse.ArgumentParser(add_help=False, prog='bigchaindb') + +base_parser.add_argument('-c', '--config', + help='Specify the location of the configuration file ' + '(use "-" for stdout)') + +# NOTE: this flag should not have any default value because that will override +# the environment variables provided to configure the logger. +base_parser.add_argument('-l', '--log-level', + type=str.upper, # convert to uppercase for comparison to choices + choices=['DEBUG', 'BENCHMARK', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + help='Log level') + +base_parser.add_argument('-y', '--yes', '--yes-please', + action='store_true', + help='Assume "yes" as answer to all prompts and run ' + 'non-interactively') + +base_parser.add_argument('-v', '--version', + action='version', + version='%(prog)s {}'.format(__version__)) diff --git a/bigchaindb/common/__init__.py b/bigchaindb/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bigchaindb/common/crypto.py b/bigchaindb/common/crypto.py new file mode 100644 index 0000000..2bf6940 --- /dev/null +++ b/bigchaindb/common/crypto.py @@ -0,0 +1,55 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Separate all crypto code so that we can easily test several implementations +from collections import namedtuple + +try: + from hashlib import sha3_256 +except ImportError: + from sha3 import sha3_256 + +from cryptoconditions import crypto + + +CryptoKeypair = namedtuple('CryptoKeypair', ('private_key', 'public_key')) + + +def hash_data(data): + """Hash the provided data using SHA3-256""" + return sha3_256(data.encode()).hexdigest() + + +def generate_key_pair(): + """Generates a cryptographic key pair. + + Returns: + :class:`~bigchaindb.common.crypto.CryptoKeypair`: A + :obj:`collections.namedtuple` with named fields + :attr:`~bigchaindb.common.crypto.CryptoKeypair.private_key` and + :attr:`~bigchaindb.common.crypto.CryptoKeypair.public_key`. + + """ + # TODO FOR CC: Adjust interface so that this function becomes unnecessary + return CryptoKeypair( + *(k.decode() for k in crypto.ed25519_generate_key_pair())) + + +PrivateKey = crypto.Ed25519SigningKey +PublicKey = crypto.Ed25519VerifyingKey + + +def key_pair_from_ed25519_key(hex_private_key): + """Generate base58 encode public-private key pair from a hex encoded private key""" + priv_key = crypto.Ed25519SigningKey(bytes.fromhex(hex_private_key)[:32], encoding='bytes') + public_key = priv_key.get_verifying_key() + return CryptoKeypair(private_key=priv_key.encode(encoding='base58').decode('utf-8'), + public_key=public_key.encode(encoding='base58').decode('utf-8')) + + +def public_key_from_ed25519_key(hex_public_key): + """Generate base58 public key from hex encoded public key""" + public_key = crypto.Ed25519VerifyingKey(bytes.fromhex(hex_public_key), encoding='bytes') + return public_key.encode(encoding='base58').decode('utf-8') diff --git a/bigchaindb/common/exceptions.py b/bigchaindb/common/exceptions.py new file mode 100644 index 0000000..750da36 --- /dev/null +++ b/bigchaindb/common/exceptions.py @@ -0,0 +1,115 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Custom exceptions used in the `bigchaindb` package. +""" +from bigchaindb.exceptions import BigchainDBError + + +class ConfigurationError(BigchainDBError): + """Raised when there is a problem with server configuration""" + + +class DatabaseDoesNotExist(BigchainDBError): + """Raised when trying to delete the database but the db is not there""" + + +class StartupError(BigchainDBError): + """Raised when there is an error starting up the system""" + + +class CyclicBlockchainError(BigchainDBError): + """Raised when there is a cycle in the blockchain""" + + +class KeypairMismatchException(BigchainDBError): + """Raised if the private key(s) provided for signing don't match any of the + current owner(s) + """ + + +class OperationError(BigchainDBError): + """Raised when an operation cannot go through""" + + +################################################################################ +# Validation errors +# +# All validation errors (which are handleable errors, not faults) should +# subclass ValidationError. However, where possible they should also have their +# own distinct type to differentiate them from other validation errors, +# especially for the purposes of testing. + + +class ValidationError(BigchainDBError): + """Raised if there was an error in validation""" + + +class DoubleSpend(ValidationError): + """Raised if a double spend is found""" + + +class InvalidHash(ValidationError): + """Raised if there was an error checking the hash for a particular + operation + """ + + +class SchemaValidationError(ValidationError): + """Raised if there was any error validating an object's schema""" + + +class InvalidSignature(ValidationError): + """Raised if there was an error checking the signature for a particular + operation + """ + + +class AssetIdMismatch(ValidationError): + """Raised when multiple transaction inputs related to different assets""" + + +class AmountError(ValidationError): + """Raised when there is a problem with a transaction's output amounts""" + + +class InputDoesNotExist(ValidationError): + """Raised if a transaction input does not exist""" + + +class TransactionOwnerError(ValidationError): + """Raised if a user tries to transfer a transaction they don't own""" + + +class DuplicateTransaction(ValidationError): + """Raised if a duplicated transaction is found""" + + +class ThresholdTooDeep(ValidationError): + """Raised if threshold condition is too deep""" + + +class MultipleValidatorOperationError(ValidationError): + """Raised when a validator update pending but new request is submited""" + + +class MultipleInputsError(ValidationError): + """Raised if there were multiple inputs when only one was expected""" + + +class InvalidProposer(ValidationError): + """Raised if the public key is not a part of the validator set""" + + +class UnequalValidatorSet(ValidationError): + """Raised if the validator sets differ""" + + +class InvalidPowerChange(ValidationError): + """Raised if proposed power change in validator set is >=1/3 total power""" + + +class InvalidPublicKey(ValidationError): + """Raised if public key doesn't match the encoding type""" diff --git a/bigchaindb/common/memoize.py b/bigchaindb/common/memoize.py new file mode 100644 index 0000000..b814e51 --- /dev/null +++ b/bigchaindb/common/memoize.py @@ -0,0 +1,58 @@ +import functools +import codecs +from functools import lru_cache + + +class HDict(dict): + def __hash__(self): + return hash(codecs.decode(self['id'], 'hex')) + + +@lru_cache(maxsize=16384) +def from_dict(func, *args, **kwargs): + return func(*args, **kwargs) + + +def memoize_from_dict(func): + + @functools.wraps(func) + def memoized_func(*args, **kwargs): + + if args[1].get('id', None): + args = list(args) + args[1] = HDict(args[1]) + new_args = tuple(args) + return from_dict(func, *new_args, **kwargs) + else: + return func(*args, **kwargs) + + return memoized_func + + +class ToDictWrapper(): + def __init__(self, tx): + self.tx = tx + + def __eq__(self, other): + return self.tx.id == other.tx.id + + def __hash__(self): + return hash(self.tx.id) + + +@lru_cache(maxsize=16384) +def to_dict(func, tx_wrapped): + return func(tx_wrapped.tx) + + +def memoize_to_dict(func): + + @functools.wraps(func) + def memoized_func(*args, **kwargs): + + if args[0].id: + return to_dict(func, ToDictWrapper(args[0])) + else: + return func(*args, **kwargs) + + return memoized_func diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md new file mode 100644 index 0000000..b06e438 --- /dev/null +++ b/bigchaindb/common/schema/README.md @@ -0,0 +1,54 @@ + + +# Introduction + +This directory contains the schemas for the different JSON documents Planetmint uses. + +The aim is to provide: + +- a strict definition of the data structures used in Planetmint, +- a language-independent tool to validate the structure of incoming/outcoming + data. (There are several ready to use + [implementations](http://json-schema.org/implementations.html) written in + different languages.) + +## Sources + +The files defining the JSON Schema for transactions (`transaction_*.yaml`) +are based on the [Planetmint Transactions Specs](https://github.com/bigchaindb/BEPs/tree/master/tx-specs). +If you want to add a new transaction version, +you must write a spec for it first. +(You can't change the JSON Schema files for old versions. +Those were used to validate old transactions +and are needed to re-check those transactions.) + +There used to be a file defining the JSON Schema for votes, named `vote.yaml`. +It was used by Planetmint version 1.3.0 and earlier. +If you want a copy of the latest `vote.yaml` file, +then you can get it from the version 1.3.0 release on GitHub, at +[https://github.com/bigchaindb/bigchaindb/blob/v1.3.0/bigchaindb/common/schema/vote.yaml](https://github.com/bigchaindb/bigchaindb/blob/v1.3.0/bigchaindb/common/schema/vote.yaml). + +## Learn about JSON Schema + +A good resource is [Understanding JSON Schema](http://spacetelescope.github.io/understanding-json-schema/index.html). +It provides a *more accessible documentation for JSON schema* than the [specs](http://json-schema.org/documentation.html). + +## If it's supposed to be JSON, why's everything in YAML D:? + +YAML is great for its conciseness and friendliness towards human-editing in comparision to JSON. + +Although YAML is a superset of JSON, at the end of the day, JSON Schema processors, like +[json-schema](http://python-jsonschema.readthedocs.io/en/latest/), take in a native object (e.g. +Python dicts or JavaScript objects) as the schema used for validation. As long as we can serialize +the YAML into what the JSON Schema processor expects (almost always as simple as loading the YAML +like you would with a JSON file), it's the same as using JSON. + +Specific advantages of using YAML: + - Legibility, especially when nesting + - Multi-line string literals, that make it easy to include descriptions that can be [auto-generated + into Sphinx documentation](/docs/server/generate_schema_documentation.py) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py new file mode 100644 index 0000000..8343fa8 --- /dev/null +++ b/bigchaindb/common/schema/__init__.py @@ -0,0 +1,82 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Schema validation related functions and data""" +import os.path +import logging + +import jsonschema +import yaml +import rapidjson + +from bigchaindb.common.exceptions import SchemaValidationError + + +logger = logging.getLogger(__name__) + + +def _load_schema(name, path=__file__): + """Load a schema from disk""" + path = os.path.join(os.path.dirname(path), name + '.yaml') + with open(path) as handle: + schema = yaml.safe_load(handle) + fast_schema = rapidjson.Validator(rapidjson.dumps(schema)) + return path, (schema, fast_schema) + + +TX_SCHEMA_VERSION = 'v2.0' + +TX_SCHEMA_PATH, TX_SCHEMA_COMMON = _load_schema('transaction_' + + TX_SCHEMA_VERSION) +_, TX_SCHEMA_CREATE = _load_schema('transaction_create_' + + TX_SCHEMA_VERSION) +_, TX_SCHEMA_TRANSFER = _load_schema('transaction_transfer_' + + TX_SCHEMA_VERSION) + +_, TX_SCHEMA_VALIDATOR_ELECTION = _load_schema('transaction_validator_election_' + + TX_SCHEMA_VERSION) + +_, TX_SCHEMA_CHAIN_MIGRATION_ELECTION = _load_schema('transaction_chain_migration_election_' + + TX_SCHEMA_VERSION) + +_, TX_SCHEMA_VOTE = _load_schema('transaction_vote_' + TX_SCHEMA_VERSION) + + +def _validate_schema(schema, body): + """Validate data against a schema""" + + # Note + # + # Schema validation is currently the major CPU bottleneck of + # Planetmint. the `jsonschema` library validates python data structures + # directly and produces nice error messages, but validation takes 4+ ms + # per transaction which is pretty slow. The rapidjson library validates + # much faster at 1.5ms, however it produces _very_ poor error messages. + # For this reason we use both, rapidjson as an optimistic pathway and + # jsonschema as a fallback in case there is a failure, so we can produce + # a helpful error message. + + try: + schema[1](rapidjson.dumps(body)) + except ValueError as exc: + try: + jsonschema.validate(body, schema[0]) + except jsonschema.ValidationError as exc2: + raise SchemaValidationError(str(exc2)) from exc2 + logger.warning('code problem: jsonschema did not raise an exception, wheras rapidjson raised %s', exc) + raise SchemaValidationError(str(exc)) from exc + + +def validate_transaction_schema(tx): + """Validate a transaction dict. + + TX_SCHEMA_COMMON contains properties that are common to all types of + transaction. TX_SCHEMA_[TRANSFER|CREATE] add additional constraints on top. + """ + _validate_schema(TX_SCHEMA_COMMON, tx) + if tx['operation'] == 'TRANSFER': + _validate_schema(TX_SCHEMA_TRANSFER, tx) + else: + _validate_schema(TX_SCHEMA_CREATE, tx) diff --git a/bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml b/bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml new file mode 100644 index 0000000..d5c5f4a --- /dev/null +++ b/bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml @@ -0,0 +1,45 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Chain Migration Election Schema - Propose a halt in block production to allow for a version change +required: +- operation +- asset +- outputs +properties: + operation: + type: string + value: "CHAIN_MIGRATION_ELECTION" + asset: + additionalProperties: false + properties: + data: + additionalProperties: false + properties: + seed: + type: string + required: + - data + outputs: + type: array + items: + "$ref": "#/definitions/output" +definitions: + output: + type: object + properties: + condition: + type: object + required: + - uri + properties: + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" diff --git a/bigchaindb/common/schema/transaction_create_v1.0.yaml b/bigchaindb/common/schema/transaction_create_v1.0.yaml new file mode 100644 index 0000000..d43b543 --- /dev/null +++ b/bigchaindb/common/schema/transaction_create_v1.0.yaml @@ -0,0 +1,35 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - CREATE/GENESIS specific constraints +required: +- asset +- inputs +properties: + asset: + additionalProperties: false + properties: + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + required: + - data + inputs: + type: array + title: "Transaction inputs" + maxItems: 1 + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "null" diff --git a/bigchaindb/common/schema/transaction_create_v2.0.yaml b/bigchaindb/common/schema/transaction_create_v2.0.yaml new file mode 100644 index 0000000..d3c7ea2 --- /dev/null +++ b/bigchaindb/common/schema/transaction_create_v2.0.yaml @@ -0,0 +1,35 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - CREATE specific constraints +required: +- asset +- inputs +properties: + asset: + additionalProperties: false + properties: + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + required: + - data + inputs: + type: array + title: "Transaction inputs" + maxItems: 1 + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "null" diff --git a/bigchaindb/common/schema/transaction_transfer_v1.0.yaml b/bigchaindb/common/schema/transaction_transfer_v1.0.yaml new file mode 100644 index 0000000..0ac4023 --- /dev/null +++ b/bigchaindb/common/schema/transaction_transfer_v1.0.yaml @@ -0,0 +1,34 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - TRANSFER specific properties +required: +- asset +properties: + asset: + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + required: + - id + inputs: + type: array + title: "Transaction inputs" + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "object" +definitions: + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string diff --git a/bigchaindb/common/schema/transaction_transfer_v2.0.yaml b/bigchaindb/common/schema/transaction_transfer_v2.0.yaml new file mode 100644 index 0000000..0ac4023 --- /dev/null +++ b/bigchaindb/common/schema/transaction_transfer_v2.0.yaml @@ -0,0 +1,34 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Transaction Schema - TRANSFER specific properties +required: +- asset +properties: + asset: + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + required: + - id + inputs: + type: array + title: "Transaction inputs" + minItems: 1 + items: + type: "object" + required: + - fulfills + properties: + fulfills: + type: "object" +definitions: + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string diff --git a/bigchaindb/common/schema/transaction_v1.0.yaml b/bigchaindb/common/schema/transaction_v1.0.yaml new file mode 100644 index 0000000..3546d78 --- /dev/null +++ b/bigchaindb/common/schema/transaction_v1.0.yaml @@ -0,0 +1,168 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +additionalProperties: false +title: Transaction Schema +required: +- id +- inputs +- outputs +- operation +- metadata +- asset +- version +properties: + id: + anyOf: + - "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" + inputs: + type: array + title: "Transaction inputs" + items: + "$ref": "#/definitions/input" + outputs: + type: array + items: + "$ref": "#/definitions/output" + metadata: + "$ref": "#/definitions/metadata" + version: + type: string + pattern: "^1\\.0$" +definitions: + offset: + type: integer + minimum: 0 + base58: + pattern: "[1-9a-zA-Z^OIl]{43,44}" + type: string + public_keys: + anyOf: + - type: array + items: + "$ref": "#/definitions/base58" + - type: 'null' + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string + uuid4: + pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" + type: string + operation: + type: string + enum: + - CREATE + - TRANSFER + - GENESIS + asset: + type: object + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + output: + type: object + additionalProperties: false + required: + - amount + - condition + - public_keys + properties: + amount: + type: string + pattern: "^[0-9]{1,20}$" + condition: + type: object + additionalProperties: false + required: + - details + - uri + properties: + details: + "$ref": "#/definitions/condition_details" + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" + public_keys: + "$ref": "#/definitions/public_keys" + input: + type: "object" + additionalProperties: false + required: + - owners_before + - fulfillment + properties: + owners_before: + "$ref": "#/definitions/public_keys" + fulfillment: + anyOf: + - type: string + pattern: "^[a-zA-Z0-9_-]*$" + - "$ref": "#/definitions/condition_details" + fulfills: + anyOf: + - type: 'object' + additionalProperties: false + required: + - output_index + - transaction_id + properties: + output_index: + "$ref": "#/definitions/offset" + transaction_id: + "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + metadata: + anyOf: + - type: object + additionalProperties: true + minProperties: 1 + - type: 'null' + condition_details: + anyOf: + - type: object + additionalProperties: false + required: + - type + - public_key + properties: + type: + type: string + pattern: "^ed25519-sha-256$" + public_key: + "$ref": "#/definitions/base58" + - type: object + additionalProperties: false + required: + - type + - threshold + - subconditions + properties: + type: + type: "string" + pattern: "^threshold-sha-256$" + threshold: + type: integer + minimum: 1 + maximum: 100 + subconditions: + type: array + items: + "$ref": "#/definitions/condition_details" diff --git a/bigchaindb/common/schema/transaction_v2.0.yaml b/bigchaindb/common/schema/transaction_v2.0.yaml new file mode 100644 index 0000000..604302f --- /dev/null +++ b/bigchaindb/common/schema/transaction_v2.0.yaml @@ -0,0 +1,170 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +additionalProperties: false +title: Transaction Schema +required: +- id +- inputs +- outputs +- operation +- metadata +- asset +- version +properties: + id: + anyOf: + - "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" + inputs: + type: array + title: "Transaction inputs" + items: + "$ref": "#/definitions/input" + outputs: + type: array + items: + "$ref": "#/definitions/output" + metadata: + "$ref": "#/definitions/metadata" + version: + type: string + pattern: "^2\\.0$" +definitions: + offset: + type: integer + minimum: 0 + base58: + pattern: "[1-9a-zA-Z^OIl]{43,44}" + type: string + public_keys: + anyOf: + - type: array + items: + "$ref": "#/definitions/base58" + - type: 'null' + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string + uuid4: + pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" + type: string + operation: + type: string + enum: + - CREATE + - TRANSFER + - VALIDATOR_ELECTION + - CHAIN_MIGRATION_ELECTION + - VOTE + asset: + type: object + additionalProperties: false + properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + data: + anyOf: + - type: object + additionalProperties: true + - type: 'null' + output: + type: object + additionalProperties: false + required: + - amount + - condition + - public_keys + properties: + amount: + type: string + pattern: "^[0-9]{1,20}$" + condition: + type: object + additionalProperties: false + required: + - details + - uri + properties: + details: + "$ref": "#/definitions/condition_details" + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" + public_keys: + "$ref": "#/definitions/public_keys" + input: + type: "object" + additionalProperties: false + required: + - owners_before + - fulfillment + properties: + owners_before: + "$ref": "#/definitions/public_keys" + fulfillment: + anyOf: + - type: string + pattern: "^[a-zA-Z0-9_-]*$" + - "$ref": "#/definitions/condition_details" + fulfills: + anyOf: + - type: 'object' + additionalProperties: false + required: + - output_index + - transaction_id + properties: + output_index: + "$ref": "#/definitions/offset" + transaction_id: + "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + metadata: + anyOf: + - type: object + additionalProperties: true + minProperties: 1 + - type: 'null' + condition_details: + anyOf: + - type: object + additionalProperties: false + required: + - type + - public_key + properties: + type: + type: string + pattern: "^ed25519-sha-256$" + public_key: + "$ref": "#/definitions/base58" + - type: object + additionalProperties: false + required: + - type + - threshold + - subconditions + properties: + type: + type: "string" + pattern: "^threshold-sha-256$" + threshold: + type: integer + minimum: 1 + maximum: 100 + subconditions: + type: array + items: + "$ref": "#/definitions/condition_details" diff --git a/bigchaindb/common/schema/transaction_validator_election_v2.0.yaml b/bigchaindb/common/schema/transaction_validator_election_v2.0.yaml new file mode 100644 index 0000000..f93353c --- /dev/null +++ b/bigchaindb/common/schema/transaction_validator_election_v2.0.yaml @@ -0,0 +1,68 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Validator Election Schema - Propose a change to validator set +required: +- operation +- asset +- outputs +properties: + operation: + type: string + value: "VALIDATOR_ELECTION" + asset: + additionalProperties: false + properties: + data: + additionalProperties: false + properties: + node_id: + type: string + seed: + type: string + public_key: + type: object + additionalProperties: false + required: + - value + - type + properties: + value: + type: string + type: + type: string + enum: + - ed25519-base16 + - ed25519-base32 + - ed25519-base64 + power: + "$ref": "#/definitions/positiveInteger" + required: + - node_id + - public_key + - power + required: + - data + outputs: + type: array + items: + "$ref": "#/definitions/output" +definitions: + output: + type: object + properties: + condition: + type: object + required: + - uri + properties: + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" diff --git a/bigchaindb/common/schema/transaction_vote_v2.0.yaml b/bigchaindb/common/schema/transaction_vote_v2.0.yaml new file mode 100644 index 0000000..64ed6ee --- /dev/null +++ b/bigchaindb/common/schema/transaction_vote_v2.0.yaml @@ -0,0 +1,34 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +"$schema": "http://json-schema.org/draft-04/schema#" +type: object +title: Vote Schema - Vote on an election +required: +- operation +- outputs +properties: + operation: + type: string + value: "VOTE" + outputs: + type: array + items: + "$ref": "#/definitions/output" +definitions: + output: + type: object + properties: + condition: + type: object + required: + - uri + properties: + uri: + type: string + pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\ + (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\ + subtypes=ed25519-sha-256(&)?){2,3}$" diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py new file mode 100644 index 0000000..432757c --- /dev/null +++ b/bigchaindb/common/transaction.py @@ -0,0 +1,1330 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Transaction related models to parse and construct transaction +payloads. + +Attributes: + UnspentOutput (namedtuple): Object holding the information + representing an unspent output. + +""" +from collections import namedtuple +from copy import deepcopy +from functools import reduce, lru_cache +import rapidjson + +import base58 +from cryptoconditions import Fulfillment, ThresholdSha256, Ed25519Sha256 +from cryptoconditions.exceptions import ( + ParsingError, ASN1DecodeError, ASN1EncodeError, UnsupportedTypeError) +try: + from hashlib import sha3_256 +except ImportError: + from sha3 import sha3_256 + +from bigchaindb.common.crypto import PrivateKey, hash_data +from bigchaindb.common.exceptions import (KeypairMismatchException, + InputDoesNotExist, DoubleSpend, + InvalidHash, InvalidSignature, + AmountError, AssetIdMismatch, + ThresholdTooDeep) +from bigchaindb.common.utils import serialize +from .memoize import memoize_from_dict, memoize_to_dict + + +UnspentOutput = namedtuple( + 'UnspentOutput', ( + # TODO 'utxo_hash': sha3_256(f'{txid}{output_index}'.encode()) + # 'utxo_hash', # noqa + 'transaction_id', + 'output_index', + 'amount', + 'asset_id', + 'condition_uri', + ) +) + + +class Input(object): + """A Input is used to spend assets locked by an Output. + + Wraps around a Crypto-condition Fulfillment. + + Attributes: + fulfillment (:class:`cryptoconditions.Fulfillment`): A Fulfillment + to be signed with a private key. + owners_before (:obj:`list` of :obj:`str`): A list of owners after a + Transaction was confirmed. + fulfills (:class:`~bigchaindb.common.transaction. TransactionLink`, + optional): A link representing the input of a `TRANSFER` + Transaction. + """ + + def __init__(self, fulfillment, owners_before, fulfills=None): + """Create an instance of an :class:`~.Input`. + + Args: + fulfillment (:class:`cryptoconditions.Fulfillment`): A + Fulfillment to be signed with a private key. + owners_before (:obj:`list` of :obj:`str`): A list of owners + after a Transaction was confirmed. + fulfills (:class:`~bigchaindb.common.transaction. + TransactionLink`, optional): A link representing the input + of a `TRANSFER` Transaction. + """ + if fulfills is not None and not isinstance(fulfills, TransactionLink): + raise TypeError('`fulfills` must be a TransactionLink instance') + if not isinstance(owners_before, list): + raise TypeError('`owners_before` must be a list instance') + + self.fulfillment = fulfillment + self.fulfills = fulfills + self.owners_before = owners_before + + def __eq__(self, other): + # TODO: If `other !== Fulfillment` return `False` + return self.to_dict() == other.to_dict() + + # NOTE: This function is used to provide a unique key for a given + # Input to suppliment memoization + def __hash__(self): + return hash((self.fulfillment, self.fulfills)) + + def to_dict(self): + """Transforms the object to a Python dictionary. + + Note: + If an Input hasn't been signed yet, this method returns a + dictionary representation. + + Returns: + dict: The Input as an alternative serialization format. + """ + try: + fulfillment = self.fulfillment.serialize_uri() + except (TypeError, AttributeError, ASN1EncodeError, ASN1DecodeError): + fulfillment = _fulfillment_to_details(self.fulfillment) + + try: + # NOTE: `self.fulfills` can be `None` and that's fine + fulfills = self.fulfills.to_dict() + except AttributeError: + fulfills = None + + input_ = { + 'owners_before': self.owners_before, + 'fulfills': fulfills, + 'fulfillment': fulfillment, + } + return input_ + + @classmethod + def generate(cls, public_keys): + # TODO: write docstring + # The amount here does not really matter. It is only use on the + # output data model but here we only care about the fulfillment + output = Output.generate(public_keys, 1) + return cls(output.fulfillment, public_keys) + + @classmethod + def from_dict(cls, data): + """Transforms a Python dictionary to an Input object. + + Note: + Optionally, this method can also serialize a Cryptoconditions- + Fulfillment that is not yet signed. + + Args: + data (dict): The Input to be transformed. + + Returns: + :class:`~bigchaindb.common.transaction.Input` + + Raises: + InvalidSignature: If an Input's URI couldn't be parsed. + """ + fulfillment = data['fulfillment'] + if not isinstance(fulfillment, (Fulfillment, type(None))): + try: + fulfillment = Fulfillment.from_uri(data['fulfillment']) + except ASN1DecodeError: + # TODO Remove as it is legacy code, and simply fall back on + # ASN1DecodeError + raise InvalidSignature("Fulfillment URI couldn't been parsed") + except TypeError: + # NOTE: See comment about this special case in + # `Input.to_dict` + fulfillment = _fulfillment_from_details(data['fulfillment']) + fulfills = TransactionLink.from_dict(data['fulfills']) + return cls(fulfillment, data['owners_before'], fulfills) + + +def _fulfillment_to_details(fulfillment): + """Encode a fulfillment as a details dictionary + + Args: + fulfillment: Crypto-conditions Fulfillment object + """ + + if fulfillment.type_name == 'ed25519-sha-256': + return { + 'type': 'ed25519-sha-256', + 'public_key': base58.b58encode(fulfillment.public_key).decode(), + } + + if fulfillment.type_name == 'threshold-sha-256': + subconditions = [ + _fulfillment_to_details(cond['body']) + for cond in fulfillment.subconditions + ] + return { + 'type': 'threshold-sha-256', + 'threshold': fulfillment.threshold, + 'subconditions': subconditions, + } + + raise UnsupportedTypeError(fulfillment.type_name) + + +def _fulfillment_from_details(data, _depth=0): + """Load a fulfillment for a signing spec dictionary + + Args: + data: tx.output[].condition.details dictionary + """ + if _depth == 100: + raise ThresholdTooDeep() + + if data['type'] == 'ed25519-sha-256': + public_key = base58.b58decode(data['public_key']) + return Ed25519Sha256(public_key=public_key) + + if data['type'] == 'threshold-sha-256': + threshold = ThresholdSha256(data['threshold']) + for cond in data['subconditions']: + cond = _fulfillment_from_details(cond, _depth+1) + threshold.add_subfulfillment(cond) + return threshold + + raise UnsupportedTypeError(data.get('type')) + + +class TransactionLink(object): + """An object for unidirectional linking to a Transaction's Output. + + Attributes: + txid (str, optional): A Transaction to link to. + output (int, optional): An output's index in a Transaction with id + `txid`. + """ + + def __init__(self, txid=None, output=None): + """Create an instance of a :class:`~.TransactionLink`. + + Note: + In an IPLD implementation, this class is not necessary anymore, + as an IPLD link can simply point to an object, as well as an + objects properties. So instead of having a (de)serializable + class, we can have a simple IPLD link of the form: + `//transaction/outputs//`. + + Args: + txid (str, optional): A Transaction to link to. + output (int, optional): An Outputs's index in a Transaction with + id `txid`. + """ + self.txid = txid + self.output = output + + def __bool__(self): + return self.txid is not None and self.output is not None + + def __eq__(self, other): + # TODO: If `other !== TransactionLink` return `False` + return self.to_dict() == other.to_dict() + + def __hash__(self): + return hash((self.txid, self.output)) + + @classmethod + def from_dict(cls, link): + """Transforms a Python dictionary to a TransactionLink object. + + Args: + link (dict): The link to be transformed. + + Returns: + :class:`~bigchaindb.common.transaction.TransactionLink` + """ + try: + return cls(link['transaction_id'], link['output_index']) + except TypeError: + return cls() + + def to_dict(self): + """Transforms the object to a Python dictionary. + + Returns: + (dict|None): The link as an alternative serialization format. + """ + if self.txid is None and self.output is None: + return None + else: + return { + 'transaction_id': self.txid, + 'output_index': self.output, + } + + def to_uri(self, path=''): + if self.txid is None and self.output is None: + return None + return '{}/transactions/{}/outputs/{}'.format(path, self.txid, + self.output) + + +class Output(object): + """An Output is used to lock an asset. + + Wraps around a Crypto-condition Condition. + + Attributes: + fulfillment (:class:`cryptoconditions.Fulfillment`): A Fulfillment + to extract a Condition from. + public_keys (:obj:`list` of :obj:`str`, optional): A list of + owners before a Transaction was confirmed. + """ + + MAX_AMOUNT = 9 * 10 ** 18 + + def __init__(self, fulfillment, public_keys=None, amount=1): + """Create an instance of a :class:`~.Output`. + + Args: + fulfillment (:class:`cryptoconditions.Fulfillment`): A + Fulfillment to extract a Condition from. + public_keys (:obj:`list` of :obj:`str`, optional): A list of + owners before a Transaction was confirmed. + amount (int): The amount of Assets to be locked with this + Output. + + Raises: + TypeError: if `public_keys` is not instance of `list`. + """ + if not isinstance(public_keys, list) and public_keys is not None: + raise TypeError('`public_keys` must be a list instance or None') + if not isinstance(amount, int): + raise TypeError('`amount` must be an int') + if amount < 1: + raise AmountError('`amount` must be greater than 0') + if amount > self.MAX_AMOUNT: + raise AmountError('`amount` must be <= %s' % self.MAX_AMOUNT) + + self.fulfillment = fulfillment + self.amount = amount + self.public_keys = public_keys + + def __eq__(self, other): + # TODO: If `other !== Condition` return `False` + return self.to_dict() == other.to_dict() + + def to_dict(self): + """Transforms the object to a Python dictionary. + + Note: + A dictionary serialization of the Input the Output was + derived from is always provided. + + Returns: + dict: The Output as an alternative serialization format. + """ + # TODO FOR CC: It must be able to recognize a hashlock condition + # and fulfillment! + condition = {} + try: + condition['details'] = _fulfillment_to_details(self.fulfillment) + except AttributeError: + pass + + try: + condition['uri'] = self.fulfillment.condition_uri + except AttributeError: + condition['uri'] = self.fulfillment + + output = { + 'public_keys': self.public_keys, + 'condition': condition, + 'amount': str(self.amount), + } + return output + + @classmethod + def generate(cls, public_keys, amount): + """Generates a Output from a specifically formed tuple or list. + + Note: + If a ThresholdCondition has to be generated where the threshold + is always the number of subconditions it is split between, a + list of the following structure is sufficient: + + [(address|condition)*, [(address|condition)*, ...], ...] + + Args: + public_keys (:obj:`list` of :obj:`str`): The public key of + the users that should be able to fulfill the Condition + that is being created. + amount (:obj:`int`): The amount locked by the Output. + + Returns: + An Output that can be used in a Transaction. + + Raises: + TypeError: If `public_keys` is not an instance of `list`. + ValueError: If `public_keys` is an empty list. + """ + threshold = len(public_keys) + if not isinstance(amount, int): + raise TypeError('`amount` must be a int') + if amount < 1: + raise AmountError('`amount` needs to be greater than zero') + if not isinstance(public_keys, list): + raise TypeError('`public_keys` must be an instance of list') + if len(public_keys) == 0: + raise ValueError('`public_keys` needs to contain at least one' + 'owner') + elif len(public_keys) == 1 and not isinstance(public_keys[0], list): + if isinstance(public_keys[0], Fulfillment): + ffill = public_keys[0] + else: + ffill = Ed25519Sha256( + public_key=base58.b58decode(public_keys[0])) + return cls(ffill, public_keys, amount=amount) + else: + initial_cond = ThresholdSha256(threshold=threshold) + threshold_cond = reduce(cls._gen_condition, public_keys, + initial_cond) + return cls(threshold_cond, public_keys, amount=amount) + + @classmethod + def _gen_condition(cls, initial, new_public_keys): + """Generates ThresholdSha256 conditions from a list of new owners. + + Note: + This method is intended only to be used with a reduce function. + For a description on how to use this method, see + :meth:`~.Output.generate`. + + Args: + initial (:class:`cryptoconditions.ThresholdSha256`): + A Condition representing the overall root. + new_public_keys (:obj:`list` of :obj:`str`|str): A list of new + owners or a single new owner. + + Returns: + :class:`cryptoconditions.ThresholdSha256`: + """ + try: + threshold = len(new_public_keys) + except TypeError: + threshold = None + + if isinstance(new_public_keys, list) and len(new_public_keys) > 1: + ffill = ThresholdSha256(threshold=threshold) + reduce(cls._gen_condition, new_public_keys, ffill) + elif isinstance(new_public_keys, list) and len(new_public_keys) <= 1: + raise ValueError('Sublist cannot contain single owner') + else: + try: + new_public_keys = new_public_keys.pop() + except AttributeError: + pass + # NOTE: Instead of submitting base58 encoded addresses, a user + # of this class can also submit fully instantiated + # Cryptoconditions. In the case of casting + # `new_public_keys` to a Ed25519Fulfillment with the + # result of a `TypeError`, we're assuming that + # `new_public_keys` is a Cryptocondition then. + if isinstance(new_public_keys, Fulfillment): + ffill = new_public_keys + else: + ffill = Ed25519Sha256( + public_key=base58.b58decode(new_public_keys)) + initial.add_subfulfillment(ffill) + return initial + + @classmethod + def from_dict(cls, data): + """Transforms a Python dictionary to an Output object. + + Note: + To pass a serialization cycle multiple times, a + Cryptoconditions Fulfillment needs to be present in the + passed-in dictionary, as Condition URIs are not serializable + anymore. + + Args: + data (dict): The dict to be transformed. + + Returns: + :class:`~bigchaindb.common.transaction.Output` + """ + try: + fulfillment = _fulfillment_from_details(data['condition']['details']) + except KeyError: + # NOTE: Hashlock condition case + fulfillment = data['condition']['uri'] + try: + amount = int(data['amount']) + except ValueError: + raise AmountError('Invalid amount: %s' % data['amount']) + return cls(fulfillment, data['public_keys'], amount) + + +class Transaction(object): + """A Transaction is used to create and transfer assets. + + Note: + For adding Inputs and Outputs, this class provides methods + to do so. + + Attributes: + operation (str): Defines the operation of the Transaction. + inputs (:obj:`list` of :class:`~bigchaindb.common. + transaction.Input`, optional): Define the assets to + spend. + outputs (:obj:`list` of :class:`~bigchaindb.common. + transaction.Output`, optional): Define the assets to lock. + asset (dict): Asset payload for this Transaction. ``CREATE`` + Transactions require a dict with a ``data`` + property while ``TRANSFER`` Transactions require a dict with a + ``id`` property. + metadata (dict): + Metadata to be stored along with the Transaction. + version (string): Defines the version number of a Transaction. + """ + + CREATE = 'CREATE' + TRANSFER = 'TRANSFER' + ALLOWED_OPERATIONS = (CREATE, TRANSFER) + VERSION = '2.0' + + def __init__(self, operation, asset, inputs=None, outputs=None, + metadata=None, version=None, hash_id=None, tx_dict=None): + """The constructor allows to create a customizable Transaction. + + Note: + When no `version` is provided, one is being + generated by this method. + + Args: + operation (str): Defines the operation of the Transaction. + asset (dict): Asset payload for this Transaction. + inputs (:obj:`list` of :class:`~bigchaindb.common. + transaction.Input`, optional): Define the assets to + outputs (:obj:`list` of :class:`~bigchaindb.common. + transaction.Output`, optional): Define the assets to + lock. + metadata (dict): Metadata to be stored along with the + Transaction. + version (string): Defines the version number of a Transaction. + hash_id (string): Hash id of the transaction. + """ + if operation not in self.ALLOWED_OPERATIONS: + allowed_ops = ', '.join(self.__class__.ALLOWED_OPERATIONS) + raise ValueError('`operation` must be one of {}' + .format(allowed_ops)) + + # Asset payloads for 'CREATE' operations must be None or + # dicts holding a `data` property. Asset payloads for 'TRANSFER' + # operations must be dicts holding an `id` property. + if (operation == self.CREATE and + asset is not None and not (isinstance(asset, dict) and 'data' in asset)): + raise TypeError(('`asset` must be None or a dict holding a `data` ' + " property instance for '{}' Transactions".format(operation))) + elif (operation == self.TRANSFER and + not (isinstance(asset, dict) and 'id' in asset)): + raise TypeError(('`asset` must be a dict holding an `id` property ' + 'for \'TRANSFER\' Transactions')) + + if outputs and not isinstance(outputs, list): + raise TypeError('`outputs` must be a list instance or None') + + if inputs and not isinstance(inputs, list): + raise TypeError('`inputs` must be a list instance or None') + + if metadata is not None and not isinstance(metadata, dict): + raise TypeError('`metadata` must be a dict or None') + + self.version = version if version is not None else self.VERSION + self.operation = operation + self.asset = asset + self.inputs = inputs or [] + self.outputs = outputs or [] + self.metadata = metadata + self._id = hash_id + self.tx_dict = tx_dict + + @property + def unspent_outputs(self): + """UnspentOutput: The outputs of this transaction, in a data + structure containing relevant information for storing them in + a UTXO set, and performing validation. + """ + if self.operation == self.CREATE: + self._asset_id = self._id + elif self.operation == self.TRANSFER: + self._asset_id = self.asset['id'] + return (UnspentOutput( + transaction_id=self._id, + output_index=output_index, + amount=output.amount, + asset_id=self._asset_id, + condition_uri=output.fulfillment.condition_uri, + ) for output_index, output in enumerate(self.outputs)) + + @property + def spent_outputs(self): + """Tuple of :obj:`dict`: Inputs of this transaction. Each input + is represented as a dictionary containing a transaction id and + output index. + """ + return ( + input_.fulfills.to_dict() + for input_ in self.inputs if input_.fulfills + ) + + @property + def serialized(self): + return Transaction._to_str(self.to_dict()) + + def _hash(self): + self._id = hash_data(self.serialized) + + @classmethod + def validate_create(cls, tx_signers, recipients, asset, metadata): + if not isinstance(tx_signers, list): + raise TypeError('`tx_signers` must be a list instance') + if not isinstance(recipients, list): + raise TypeError('`recipients` must be a list instance') + if len(tx_signers) == 0: + raise ValueError('`tx_signers` list cannot be empty') + if len(recipients) == 0: + raise ValueError('`recipients` list cannot be empty') + if not (asset is None or isinstance(asset, dict)): + raise TypeError('`asset` must be a dict or None') + if not (metadata is None or isinstance(metadata, dict)): + raise TypeError('`metadata` must be a dict or None') + + inputs = [] + outputs = [] + + # generate_outputs + for recipient in recipients: + if not isinstance(recipient, tuple) or len(recipient) != 2: + raise ValueError(('Each `recipient` in the list must be a' + ' tuple of `([],' + ' )`')) + pub_keys, amount = recipient + outputs.append(Output.generate(pub_keys, amount)) + + # generate inputs + inputs.append(Input.generate(tx_signers)) + + return (inputs, outputs) + + @classmethod + def create(cls, tx_signers, recipients, metadata=None, asset=None): + """A simple way to generate a `CREATE` transaction. + + Note: + This method currently supports the following Cryptoconditions + use cases: + - Ed25519 + - ThresholdSha256 + + Additionally, it provides support for the following Planetmint + use cases: + - Multiple inputs and outputs. + + Args: + tx_signers (:obj:`list` of :obj:`str`): A list of keys that + represent the signers of the CREATE Transaction. + recipients (:obj:`list` of :obj:`tuple`): A list of + ([keys],amount) that represent the recipients of this + Transaction. + metadata (dict): The metadata to be stored along with the + Transaction. + asset (dict): The metadata associated with the asset that will + be created in this Transaction. + + Returns: + :class:`~bigchaindb.common.transaction.Transaction` + """ + + (inputs, outputs) = cls.validate_create(tx_signers, recipients, asset, metadata) + return cls(cls.CREATE, {'data': asset}, inputs, outputs, metadata) + + @classmethod + def validate_transfer(cls, inputs, recipients, asset_id, metadata): + if not isinstance(inputs, list): + raise TypeError('`inputs` must be a list instance') + if len(inputs) == 0: + raise ValueError('`inputs` must contain at least one item') + if not isinstance(recipients, list): + raise TypeError('`recipients` must be a list instance') + if len(recipients) == 0: + raise ValueError('`recipients` list cannot be empty') + + outputs = [] + for recipient in recipients: + if not isinstance(recipient, tuple) or len(recipient) != 2: + raise ValueError(('Each `recipient` in the list must be a' + ' tuple of `([],' + ' )`')) + pub_keys, amount = recipient + outputs.append(Output.generate(pub_keys, amount)) + + if not isinstance(asset_id, str): + raise TypeError('`asset_id` must be a string') + + return (deepcopy(inputs), outputs) + + @classmethod + def transfer(cls, inputs, recipients, asset_id, metadata=None): + """A simple way to generate a `TRANSFER` transaction. + + Note: + Different cases for threshold conditions: + + Combining multiple `inputs` with an arbitrary number of + `recipients` can yield interesting cases for the creation of + threshold conditions we'd like to support. The following + notation is proposed: + + 1. The index of a `recipient` corresponds to the index of + an input: + e.g. `transfer([input1], [a])`, means `input1` would now be + owned by user `a`. + + 2. `recipients` can (almost) get arbitrary deeply nested, + creating various complex threshold conditions: + e.g. `transfer([inp1, inp2], [[a, [b, c]], d])`, means + `a`'s signature would have a 50% weight on `inp1` + compared to `b` and `c` that share 25% of the leftover + weight respectively. `inp2` is owned completely by `d`. + + Args: + inputs (:obj:`list` of :class:`~bigchaindb.common.transaction. + Input`): Converted `Output`s, intended to + be used as inputs in the transfer to generate. + recipients (:obj:`list` of :obj:`tuple`): A list of + ([keys],amount) that represent the recipients of this + Transaction. + asset_id (str): The asset ID of the asset to be transferred in + this Transaction. + metadata (dict): Python dictionary to be stored along with the + Transaction. + + Returns: + :class:`~bigchaindb.common.transaction.Transaction` + """ + (inputs, outputs) = cls.validate_transfer(inputs, recipients, asset_id, metadata) + return cls(cls.TRANSFER, {'id': asset_id}, inputs, outputs, metadata) + + def __eq__(self, other): + try: + other = other.to_dict() + except AttributeError: + return False + return self.to_dict() == other + + def to_inputs(self, indices=None): + """Converts a Transaction's outputs to spendable inputs. + + Note: + Takes the Transaction's outputs and derives inputs + from that can then be passed into `Transaction.transfer` as + `inputs`. + A list of integers can be passed to `indices` that + defines which outputs should be returned as inputs. + If no `indices` are passed (empty list or None) all + outputs of the Transaction are returned. + + Args: + indices (:obj:`list` of int): Defines which + outputs should be returned as inputs. + + Returns: + :obj:`list` of :class:`~bigchaindb.common.transaction. + Input` + """ + # NOTE: If no indices are passed, we just assume to take all outputs + # as inputs. + indices = indices or range(len(self.outputs)) + return [ + Input(self.outputs[idx].fulfillment, + self.outputs[idx].public_keys, + TransactionLink(self.id, idx)) + for idx in indices + ] + + def add_input(self, input_): + """Adds an input to a Transaction's list of inputs. + + Args: + input_ (:class:`~bigchaindb.common.transaction. + Input`): An Input to be added to the Transaction. + """ + if not isinstance(input_, Input): + raise TypeError('`input_` must be a Input instance') + self.inputs.append(input_) + + def add_output(self, output): + """Adds an output to a Transaction's list of outputs. + + Args: + output (:class:`~bigchaindb.common.transaction. + Output`): An Output to be added to the + Transaction. + """ + if not isinstance(output, Output): + raise TypeError('`output` must be an Output instance or None') + self.outputs.append(output) + + def sign(self, private_keys): + """Fulfills a previous Transaction's Output by signing Inputs. + + Note: + This method works only for the following Cryptoconditions + currently: + - Ed25519Fulfillment + - ThresholdSha256 + Furthermore, note that all keys required to fully sign the + Transaction have to be passed to this method. A subset of all + will cause this method to fail. + + Args: + private_keys (:obj:`list` of :obj:`str`): A complete list of + all private keys needed to sign all Fulfillments of this + Transaction. + + Returns: + :class:`~bigchaindb.common.transaction.Transaction` + """ + # TODO: Singing should be possible with at least one of all private + # keys supplied to this method. + if private_keys is None or not isinstance(private_keys, list): + raise TypeError('`private_keys` must be a list instance') + + # NOTE: Generate public keys from private keys and match them in a + # dictionary: + # key: public_key + # value: private_key + def gen_public_key(private_key): + # TODO FOR CC: Adjust interface so that this function becomes + # unnecessary + + # cc now provides a single method `encode` to return the key + # in several different encodings. + public_key = private_key.get_verifying_key().encode() + # Returned values from cc are always bytestrings so here we need + # to decode to convert the bytestring into a python str + return public_key.decode() + + key_pairs = {gen_public_key(PrivateKey(private_key)): + PrivateKey(private_key) for private_key in private_keys} + + tx_dict = self.to_dict() + tx_dict = Transaction._remove_signatures(tx_dict) + tx_serialized = Transaction._to_str(tx_dict) + for i, input_ in enumerate(self.inputs): + self.inputs[i] = self._sign_input(input_, tx_serialized, key_pairs) + + self._hash() + + return self + + @classmethod + def _sign_input(cls, input_, message, key_pairs): + """Signs a single Input. + + Note: + This method works only for the following Cryptoconditions + currently: + - Ed25519Fulfillment + - ThresholdSha256. + + Args: + input_ (:class:`~bigchaindb.common.transaction. + Input`) The Input to be signed. + message (str): The message to be signed + key_pairs (dict): The keys to sign the Transaction with. + """ + if isinstance(input_.fulfillment, Ed25519Sha256): + return cls._sign_simple_signature_fulfillment(input_, message, + key_pairs) + elif isinstance(input_.fulfillment, ThresholdSha256): + return cls._sign_threshold_signature_fulfillment(input_, message, + key_pairs) + else: + raise ValueError( + 'Fulfillment couldn\'t be matched to ' + 'Cryptocondition fulfillment type.') + + @classmethod + def _sign_simple_signature_fulfillment(cls, input_, message, key_pairs): + """Signs a Ed25519Fulfillment. + + Args: + input_ (:class:`~bigchaindb.common.transaction. + Input`) The input to be signed. + message (str): The message to be signed + key_pairs (dict): The keys to sign the Transaction with. + """ + # NOTE: To eliminate the dangers of accidentally signing a condition by + # reference, we remove the reference of input_ here + # intentionally. If the user of this class knows how to use it, + # this should never happen, but then again, never say never. + input_ = deepcopy(input_) + public_key = input_.owners_before[0] + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + + try: + # cryptoconditions makes no assumptions of the encoding of the + # message to sign or verify. It only accepts bytestrings + input_.fulfillment.sign( + message.digest(), base58.b58decode(key_pairs[public_key].encode())) + except KeyError: + raise KeypairMismatchException('Public key {} is not a pair to ' + 'any of the private keys' + .format(public_key)) + return input_ + + @classmethod + def _sign_threshold_signature_fulfillment(cls, input_, message, key_pairs): + """Signs a ThresholdSha256. + + Args: + input_ (:class:`~bigchaindb.common.transaction. + Input`) The Input to be signed. + message (str): The message to be signed + key_pairs (dict): The keys to sign the Transaction with. + """ + input_ = deepcopy(input_) + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + + for owner_before in set(input_.owners_before): + # TODO: CC should throw a KeypairMismatchException, instead of + # our manual mapping here + + # TODO FOR CC: Naming wise this is not so smart, + # `get_subcondition` in fact doesn't return a + # condition but a fulfillment + + # TODO FOR CC: `get_subcondition` is singular. One would not + # expect to get a list back. + ccffill = input_.fulfillment + subffills = ccffill.get_subcondition_from_vk( + base58.b58decode(owner_before)) + if not subffills: + raise KeypairMismatchException('Public key {} cannot be found ' + 'in the fulfillment' + .format(owner_before)) + try: + private_key = key_pairs[owner_before] + except KeyError: + raise KeypairMismatchException('Public key {} is not a pair ' + 'to any of the private keys' + .format(owner_before)) + + # cryptoconditions makes no assumptions of the encoding of the + # message to sign or verify. It only accepts bytestrings + for subffill in subffills: + subffill.sign( + message.digest(), base58.b58decode(private_key.encode())) + return input_ + + def inputs_valid(self, outputs=None): + """Validates the Inputs in the Transaction against given + Outputs. + + Note: + Given a `CREATE` Transaction is passed, + dummy values for Outputs are submitted for validation that + evaluate parts of the validation-checks to `True`. + + Args: + outputs (:obj:`list` of :class:`~bigchaindb.common. + transaction.Output`): A list of Outputs to check the + Inputs against. + + Returns: + bool: If all Inputs are valid. + """ + if self.operation == self.CREATE: + # NOTE: Since in the case of a `CREATE`-transaction we do not have + # to check for outputs, we're just submitting dummy + # values to the actual method. This simplifies it's logic + # greatly, as we do not have to check against `None` values. + return self._inputs_valid(['dummyvalue' + for _ in self.inputs]) + elif self.operation == self.TRANSFER: + return self._inputs_valid([output.fulfillment.condition_uri + for output in outputs]) + else: + allowed_ops = ', '.join(self.__class__.ALLOWED_OPERATIONS) + raise TypeError('`operation` must be one of {}' + .format(allowed_ops)) + + def _inputs_valid(self, output_condition_uris): + """Validates an Input against a given set of Outputs. + + Note: + The number of `output_condition_uris` must be equal to the + number of Inputs a Transaction has. + + Args: + output_condition_uris (:obj:`list` of :obj:`str`): A list of + Outputs to check the Inputs against. + + Returns: + bool: If all Outputs are valid. + """ + + if len(self.inputs) != len(output_condition_uris): + raise ValueError('Inputs and ' + 'output_condition_uris must have the same count') + + tx_dict = self.tx_dict if self.tx_dict else self.to_dict() + tx_dict = Transaction._remove_signatures(tx_dict) + tx_dict['id'] = None + tx_serialized = Transaction._to_str(tx_dict) + + def validate(i, output_condition_uri=None): + """Validate input against output condition URI""" + return self._input_valid(self.inputs[i], self.operation, + tx_serialized, output_condition_uri) + + return all(validate(i, cond) + for i, cond in enumerate(output_condition_uris)) + + @lru_cache(maxsize=16384) + def _input_valid(self, input_, operation, message, output_condition_uri=None): + """Validates a single Input against a single Output. + + Note: + In case of a `CREATE` Transaction, this method + does not validate against `output_condition_uri`. + + Args: + input_ (:class:`~bigchaindb.common.transaction. + Input`) The Input to be signed. + operation (str): The type of Transaction. + message (str): The fulfillment message. + output_condition_uri (str, optional): An Output to check the + Input against. + + Returns: + bool: If the Input is valid. + """ + ccffill = input_.fulfillment + try: + parsed_ffill = Fulfillment.from_uri(ccffill.serialize_uri()) + except (TypeError, ValueError, + ParsingError, ASN1DecodeError, ASN1EncodeError): + return False + + if operation == self.CREATE: + # NOTE: In the case of a `CREATE` transaction, the + # output is always valid. + output_valid = True + else: + output_valid = output_condition_uri == ccffill.condition_uri + + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + + # NOTE: We pass a timestamp to `.validate`, as in case of a timeout + # condition we'll have to validate against it + + # cryptoconditions makes no assumptions of the encoding of the + # message to sign or verify. It only accepts bytestrings + ffill_valid = parsed_ffill.validate(message=message.digest()) + return output_valid and ffill_valid + + # This function is required by `lru_cache` to create a key for memoization + def __hash__(self): + return hash(self.id) + + @memoize_to_dict + def to_dict(self): + """Transforms the object to a Python dictionary. + + Returns: + dict: The Transaction as an alternative serialization format. + """ + return { + 'inputs': [input_.to_dict() for input_ in self.inputs], + 'outputs': [output.to_dict() for output in self.outputs], + 'operation': str(self.operation), + 'metadata': self.metadata, + 'asset': self.asset, + 'version': self.version, + 'id': self._id, + } + + @staticmethod + # TODO: Remove `_dict` prefix of variable. + def _remove_signatures(tx_dict): + """Takes a Transaction dictionary and removes all signatures. + + Args: + tx_dict (dict): The Transaction to remove all signatures from. + + Returns: + dict + + """ + # NOTE: We remove the reference since we need `tx_dict` only for the + # transaction's hash + tx_dict = deepcopy(tx_dict) + for input_ in tx_dict['inputs']: + # NOTE: Not all Cryptoconditions return a `signature` key (e.g. + # ThresholdSha256), so setting it to `None` in any + # case could yield incorrect signatures. This is why we only + # set it to `None` if it's set in the dict. + input_['fulfillment'] = None + return tx_dict + + @staticmethod + def _to_hash(value): + return hash_data(value) + + @property + def id(self): + return self._id + + def to_hash(self): + return self.to_dict()['id'] + + @staticmethod + def _to_str(value): + return serialize(value) + + # TODO: This method shouldn't call `_remove_signatures` + def __str__(self): + tx = Transaction._remove_signatures(self.to_dict()) + return Transaction._to_str(tx) + + @classmethod + def get_asset_id(cls, transactions): + """Get the asset id from a list of :class:`~.Transactions`. + + This is useful when we want to check if the multiple inputs of a + transaction are related to the same asset id. + + Args: + transactions (:obj:`list` of :class:`~bigchaindb.common. + transaction.Transaction`): A list of Transactions. + Usually input Transactions that should have a matching + asset ID. + + Returns: + str: ID of the asset. + + Raises: + :exc:`AssetIdMismatch`: If the inputs are related to different + assets. + """ + + if not isinstance(transactions, list): + transactions = [transactions] + + # create a set of the transactions' asset ids + asset_ids = {tx.id if tx.operation == tx.CREATE + else tx.asset['id'] + for tx in transactions} + + # check that all the transasctions have the same asset id + if len(asset_ids) > 1: + raise AssetIdMismatch(('All inputs of all transactions passed' + ' need to have the same asset id')) + return asset_ids.pop() + + @staticmethod + def validate_id(tx_body): + """Validate the transaction ID of a transaction + + Args: + tx_body (dict): The Transaction to be transformed. + """ + # NOTE: Remove reference to avoid side effects + # tx_body = deepcopy(tx_body) + tx_body = rapidjson.loads(rapidjson.dumps(tx_body)) + + try: + proposed_tx_id = tx_body['id'] + except KeyError: + raise InvalidHash('No transaction id found!') + + tx_body['id'] = None + + tx_body_serialized = Transaction._to_str(tx_body) + valid_tx_id = Transaction._to_hash(tx_body_serialized) + + if proposed_tx_id != valid_tx_id: + err_msg = ("The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid.") + raise InvalidHash(err_msg.format(proposed_tx_id)) + + @classmethod + @memoize_from_dict + def from_dict(cls, tx, skip_schema_validation=True): + """Transforms a Python dictionary to a Transaction object. + + Args: + tx_body (dict): The Transaction to be transformed. + + Returns: + :class:`~bigchaindb.common.transaction.Transaction` + """ + operation = tx.get('operation', Transaction.CREATE) if isinstance(tx, dict) else Transaction.CREATE + cls = Transaction.resolve_class(operation) + + if not skip_schema_validation: + cls.validate_id(tx) + cls.validate_schema(tx) + + inputs = [Input.from_dict(input_) for input_ in tx['inputs']] + outputs = [Output.from_dict(output) for output in tx['outputs']] + return cls(tx['operation'], tx['asset'], inputs, outputs, + tx['metadata'], tx['version'], hash_id=tx['id'], tx_dict=tx) + + @classmethod + def from_db(cls, bigchain, tx_dict_list): + """Helper method that reconstructs a transaction dict that was returned + from the database. It checks what asset_id to retrieve, retrieves the + asset from the asset table and reconstructs the transaction. + + Args: + bigchain (:class:`~bigchaindb.tendermint.BigchainDB`): An instance + of Planetmint used to perform database queries. + tx_dict_list (:list:`dict` or :obj:`dict`): The transaction dict or + list of transaction dict as returned from the database. + + Returns: + :class:`~Transaction` + + """ + return_list = True + if isinstance(tx_dict_list, dict): + tx_dict_list = [tx_dict_list] + return_list = False + + tx_map = {} + tx_ids = [] + for tx in tx_dict_list: + tx.update({'metadata': None}) + tx_map[tx['id']] = tx + tx_ids.append(tx['id']) + + assets = list(bigchain.get_assets(tx_ids)) + for asset in assets: + if asset is not None: + tx = tx_map[asset['id']] + del asset['id'] + tx['asset'] = asset + + tx_ids = list(tx_map.keys()) + metadata_list = list(bigchain.get_metadata(tx_ids)) + for metadata in metadata_list: + tx = tx_map[metadata['id']] + tx.update({'metadata': metadata.get('metadata')}) + + if return_list: + tx_list = [] + for tx_id, tx in tx_map.items(): + tx_list.append(cls.from_dict(tx)) + return tx_list + else: + tx = list(tx_map.values())[0] + return cls.from_dict(tx) + + type_registry = {} + + @staticmethod + def register_type(tx_type, tx_class): + Transaction.type_registry[tx_type] = tx_class + + def resolve_class(operation): + """For the given `tx` based on the `operation` key return its implementation class""" + + create_txn_class = Transaction.type_registry.get(Transaction.CREATE) + return Transaction.type_registry.get(operation, create_txn_class) + + @classmethod + def validate_schema(cls, tx): + pass + + def validate_transfer_inputs(self, bigchain, current_transactions=[]): + # store the inputs so that we can check if the asset ids match + input_txs = [] + input_conditions = [] + for input_ in self.inputs: + input_txid = input_.fulfills.txid + input_tx = bigchain.get_transaction(input_txid) + + if input_tx is None: + for ctxn in current_transactions: + if ctxn.id == input_txid: + input_tx = ctxn + + if input_tx is None: + raise InputDoesNotExist("input `{}` doesn't exist" + .format(input_txid)) + + spent = bigchain.get_spent(input_txid, input_.fulfills.output, + current_transactions) + if spent: + raise DoubleSpend('input `{}` was already spent' + .format(input_txid)) + + output = input_tx.outputs[input_.fulfills.output] + input_conditions.append(output) + input_txs.append(input_tx) + + # Validate that all inputs are distinct + links = [i.fulfills.to_uri() for i in self.inputs] + if len(links) != len(set(links)): + raise DoubleSpend('tx "{}" spends inputs twice'.format(self.id)) + + # validate asset id + asset_id = self.get_asset_id(input_txs) + if asset_id != self.asset['id']: + raise AssetIdMismatch(('The asset id of the input does not' + ' match the asset id of the' + ' transaction')) + + input_amount = sum([input_condition.amount for input_condition in input_conditions]) + output_amount = sum([output_condition.amount for output_condition in self.outputs]) + + if output_amount != input_amount: + raise AmountError(('The amount used in the inputs `{}`' + ' needs to be same as the amount used' + ' in the outputs `{}`') + .format(input_amount, output_amount)) + + if not self.inputs_valid(input_conditions): + raise InvalidSignature('Transaction signature is invalid.') + + return True diff --git a/bigchaindb/common/transaction_mode_types.py b/bigchaindb/common/transaction_mode_types.py new file mode 100644 index 0000000..840dff7 --- /dev/null +++ b/bigchaindb/common/transaction_mode_types.py @@ -0,0 +1,8 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +BROADCAST_TX_COMMIT = 'broadcast_tx_commit' +BROADCAST_TX_ASYNC = 'broadcast_tx_async' +BROADCAST_TX_SYNC = 'broadcast_tx_sync' diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py new file mode 100644 index 0000000..ecb03a0 --- /dev/null +++ b/bigchaindb/common/utils.py @@ -0,0 +1,165 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import time +import re +import rapidjson + +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + + +def gen_timestamp(): + """The Unix time, rounded to the nearest second. + See https://en.wikipedia.org/wiki/Unix_time + + Returns: + str: the Unix time + """ + return str(round(time.time())) + + +def serialize(data): + """Serialize a dict into a JSON formatted string. + + This function enforces rules like the separator and order of keys. + This ensures that all dicts are serialized in the same way. + + This is specially important for hashing data. We need to make sure that + everyone serializes their data in the same way so that we do not have + hash mismatches for the same structure due to serialization + differences. + + Args: + data (dict): dict to serialize + + Returns: + str: JSON formatted string + + """ + return rapidjson.dumps(data, skipkeys=False, ensure_ascii=False, + sort_keys=True) + + +def deserialize(data): + """Deserialize a JSON formatted string into a dict. + + Args: + data (str): JSON formatted string. + + Returns: + dict: dict resulting from the serialization of a JSON formatted + string. + """ + return rapidjson.loads(data) + + +def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validate value of `key` in `obj` using `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictionary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successful + + Raises: + ValidationError: `validation_fun` will raise exception on failure + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'localmongodb': + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_keys_in_obj(obj_name, data, validation_fun) + elif isinstance(data, list): + validate_all_items_in_list(obj_name, data, validation_fun) + + +def validate_all_items_in_list(obj_name, data, validation_fun): + for item in data: + if isinstance(item, dict): + validate_all_keys_in_obj(obj_name, item, validation_fun) + elif isinstance(item, list): + validate_all_items_in_list(obj_name, item, validation_fun) + + +def validate_all_keys_in_obj(obj_name, obj, validation_fun): + """Validate all (nested) keys in `obj` by using `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictionary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successful + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for key, value in obj.items(): + validation_fun(obj_name, key) + if isinstance(value, dict): + validate_all_keys_in_obj(obj_name, value, validation_fun) + elif isinstance(value, list): + validate_all_items_in_list(obj_name, value, validation_fun) + + +def validate_all_values_for_key_in_obj(obj, key, validation_fun): + """Validate value for all (nested) occurrence of `key` in `obj` + using `validation_fun`. + + Args: + obj (dict): dictionary object. + key (str): key whose value is to be validated. + validation_fun (function): function used to validate the value + of `key`. + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for vkey, value in obj.items(): + if vkey == key: + validation_fun(value) + elif isinstance(value, dict): + validate_all_values_for_key_in_obj(value, key, validation_fun) + elif isinstance(value, list): + validate_all_values_for_key_in_list(value, key, validation_fun) + + +def validate_all_values_for_key_in_list(input_list, key, validation_fun): + for item in input_list: + if isinstance(item, dict): + validate_all_values_for_key_in_obj(item, key, validation_fun) + elif isinstance(item, list): + validate_all_values_for_key_in_list(item, key, validation_fun) + + +def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters. + + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: validation successful + + Raises: + ValidationError: will raise exception in case of regex match. + """ + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py new file mode 100644 index 0000000..aa330f4 --- /dev/null +++ b/bigchaindb/config_utils.py @@ -0,0 +1,308 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Utils for reading and setting configuration settings. + +The value of each Planetmint Server configuration setting is +determined according to the following rules: + +* If it's set by an environment variable, then use that value +* Otherwise, if it's set in a local config file, then use that + value +* Otherwise, use the default value (contained in + ``bigchaindb.__init__``) +""" + + +import os +import copy +import json +import logging +import collections.abc +from functools import lru_cache + +from pkg_resources import iter_entry_points, ResolutionError + +from bigchaindb.common import exceptions + +import bigchaindb + +from bigchaindb.validation import BaseValidationRules + +# TODO: move this to a proper configuration file for logging +logging.getLogger('requests').setLevel(logging.WARNING) +logger = logging.getLogger(__name__) + +CONFIG_DEFAULT_PATH = os.environ.setdefault( + 'PLANETMINT_CONFIG_PATH', + os.path.join(os.path.expanduser('~'), '.bigchaindb'), +) + +CONFIG_PREFIX = 'PLANETMINT' +CONFIG_SEP = '_' + + +def map_leafs(func, mapping): + """Map a function to the leafs of a mapping.""" + + def _inner(mapping, path=None): + if path is None: + path = [] + + for key, val in mapping.items(): + if isinstance(val, collections.abc.Mapping): + _inner(val, path + [key]) + else: + mapping[key] = func(val, path=path+[key]) + + return mapping + + return _inner(copy.deepcopy(mapping)) + + +# Thanks Alex <3 +# http://stackoverflow.com/a/3233356/597097 +def update(d, u): + """Recursively update a mapping (i.e. a dict, list, set, or tuple). + + Conceptually, d and u are two sets trees (with nodes and edges). + This function goes through all the nodes of u. For each node in u, + if d doesn't have that node yet, then this function adds the node from u, + otherwise this function overwrites the node already in d with u's node. + + Args: + d (mapping): The mapping to overwrite and add to. + u (mapping): The mapping to read for changes. + + Returns: + mapping: An updated version of d (updated by u). + """ + for k, v in u.items(): + if isinstance(v, collections.abc.Mapping): + r = update(d.get(k, {}), v) + d[k] = r + else: + d[k] = u[k] + return d + + +def file_config(filename=None): + """Returns the config values found in a configuration file. + + Args: + filename (str): the JSON file with the configuration values. + If ``None``, CONFIG_DEFAULT_PATH will be used. + + Returns: + dict: The config values in the specified config file (or the + file at CONFIG_DEFAULT_PATH, if filename == None) + """ + logger.debug('On entry into file_config(), filename = {}'.format(filename)) + + if filename is None: + filename = CONFIG_DEFAULT_PATH + + logger.debug('file_config() will try to open `{}`'.format(filename)) + with open(filename) as f: + try: + config = json.load(f) + except ValueError as err: + raise exceptions.ConfigurationError( + 'Failed to parse the JSON configuration from `{}`, {}'.format(filename, err) + ) + + logger.info('Configuration loaded from `{}`'.format(filename)) + + return config + + +def env_config(config): + """Return a new configuration with the values found in the environment. + + The function recursively iterates over the config, checking if there is + a matching env variable. If an env variable is found, the func updates + the configuration with that value. + + The name of the env variable is built combining a prefix (``PLANETMINT``) + with the path to the value. If the ``config`` in input is: + ``{'database': {'host': 'localhost'}}`` + this function will try to read the env variable ``PLANETMINT_DATABASE_HOST``. + """ + + def load_from_env(value, path): + var_name = CONFIG_SEP.join([CONFIG_PREFIX] + list(map(lambda s: s.upper(), path))) + + return os.environ.get(var_name, value) + + return map_leafs(load_from_env, config) + + +def update_types(config, reference, list_sep=':'): + """Return a new configuration where all the values types + are aligned with the ones in the default configuration + """ + + def _coerce(current, value): + # Coerce a value to the `current` type. + try: + # First we try to apply current to the value, since it + # might be a function + return current(value) + except TypeError: + # Then we check if current is a list AND if the value + # is a string. + if isinstance(current, list) and isinstance(value, str): + # If so, we use the colon as the separator + return value.split(list_sep) + + try: + # If we are here, we should try to apply the type + # of `current` to the value + return type(current)(value) + except TypeError: + # Worst case scenario we return the value itself. + return value + + def _update_type(value, path): + current = reference + + for elem in path: + try: + current = current[elem] + except KeyError: + return value + + return _coerce(current, value) + + return map_leafs(_update_type, config) + + +def set_config(config): + """Set bigchaindb.config equal to the default config dict, + then update that with whatever is in the provided config dict, + and then set bigchaindb.config['CONFIGURED'] = True + + Args: + config (dict): the config dict to read for changes + to the default config + + Note: + Any previous changes made to ``bigchaindb.config`` will be lost. + """ + # Deep copy the default config into bigchaindb.config + bigchaindb.config = copy.deepcopy(bigchaindb._config) + # Update the default config with whatever is in the passed config + update(bigchaindb.config, update_types(config, bigchaindb.config)) + bigchaindb.config['CONFIGURED'] = True + + +def update_config(config): + """Update bigchaindb.config with whatever is in the provided config dict, + and then set bigchaindb.config['CONFIGURED'] = True + + Args: + config (dict): the config dict to read for changes + to the default config + """ + + # Update the default config with whatever is in the passed config + update(bigchaindb.config, update_types(config, bigchaindb.config)) + bigchaindb.config['CONFIGURED'] = True + + +def write_config(config, filename=None): + """Write the provided configuration to a specific location. + + Args: + config (dict): a dictionary with the configuration to load. + filename (str): the name of the file that will store the new configuration. Defaults to ``None``. + If ``None``, the HOME of the current user and the string ``.bigchaindb`` will be used. + """ + if not filename: + filename = CONFIG_DEFAULT_PATH + + with open(filename, 'w') as f: + json.dump(config, f, indent=4) + + +def is_configured(): + return bool(bigchaindb.config.get('CONFIGURED')) + + +def autoconfigure(filename=None, config=None, force=False): + """Run ``file_config`` and ``env_config`` if the module has not + been initialized. + """ + if not force and is_configured(): + logger.debug('System already configured, skipping autoconfiguration') + return + + # start with the current configuration + newconfig = bigchaindb.config + + # update configuration from file + try: + newconfig = update(newconfig, file_config(filename=filename)) + except FileNotFoundError as e: + if filename: + raise + else: + logger.info('Cannot find config file `%s`.' % e.filename) + + # override configuration with env variables + newconfig = env_config(newconfig) + if config: + newconfig = update(newconfig, config) + set_config(newconfig) # sets bigchaindb.config + + +@lru_cache() +def load_validation_plugin(name=None): + """Find and load the chosen validation plugin. + + Args: + name (string): the name of the entry_point, as advertised in the + setup.py of the providing package. + + Returns: + an uninstantiated subclass of ``bigchaindb.validation.AbstractValidationRules`` + """ + if not name: + return BaseValidationRules + + # TODO: This will return the first plugin with group `bigchaindb.validation` + # and name `name` in the active WorkingSet. + # We should probably support Requirements specs in the config, e.g. + # validation_plugin: 'my-plugin-package==0.0.1;default' + plugin = None + for entry_point in iter_entry_points('bigchaindb.validation', name): + plugin = entry_point.load() + + # No matching entry_point found + if not plugin: + raise ResolutionError( + 'No plugin found in group `bigchaindb.validation` with name `{}`'. + format(name)) + + # Is this strictness desireable? + # It will probably reduce developer headaches in the wild. + if not issubclass(plugin, (BaseValidationRules,)): + raise TypeError('object of type "{}" does not implement `bigchaindb.' + 'validation.BaseValidationRules`'.format(type(plugin))) + + return plugin + + +def load_events_plugins(names=None): + plugins = [] + + if names is None: + return plugins + + for name in names: + for entry_point in iter_entry_points('bigchaindb.events', name): + plugins.append((name, entry_point.load())) + + return plugins diff --git a/bigchaindb/core.py b/bigchaindb/core.py new file mode 100644 index 0000000..fe4e0d6 --- /dev/null +++ b/bigchaindb/core.py @@ -0,0 +1,271 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module contains all the goodness to integrate Planetmint +with Tendermint. +""" +import logging +import sys + +from abci.application import BaseApplication +from abci import CodeTypeOk + +from bigchaindb import BigchainDB +from bigchaindb.elections.election import Election +from bigchaindb.version import __tm_supported_versions__ +from bigchaindb.utils import tendermint_version_is_compatible +from bigchaindb.tendermint_utils import (decode_transaction, + calculate_hash) +from bigchaindb.lib import Block +import bigchaindb.upsert_validator.validator_utils as vutils +from bigchaindb.events import EventTypes, Event + + +CodeTypeError = 1 +logger = logging.getLogger(__name__) + + +class App(BaseApplication): + """Bridge between Planetmint and Tendermint. + + The role of this class is to expose the Planetmint + transaction logic to Tendermint Core. + """ + + def __init__(self, abci, bigchaindb=None, events_queue=None,): + super().__init__(abci) + self.events_queue = events_queue + self.bigchaindb = bigchaindb or BigchainDB() + self.block_txn_ids = [] + self.block_txn_hash = '' + self.block_transactions = [] + self.validators = None + self.new_height = None + self.chain = self.bigchaindb.get_latest_abci_chain() + + def log_abci_migration_error(self, chain_id, validators): + logger.error('An ABCI chain migration is in process. ' + 'Download the new ABCI client and configure it with ' + f'chain_id={chain_id} and validators={validators}.') + + def abort_if_abci_chain_is_not_synced(self): + if self.chain is None or self.chain['is_synced']: + return + + validators = self.bigchaindb.get_validators() + self.log_abci_migration_error(self.chain['chain_id'], validators) + sys.exit(1) + + def init_chain(self, genesis): + """Initialize chain upon genesis or a migration""" + + app_hash = '' + height = 0 + + known_chain = self.bigchaindb.get_latest_abci_chain() + if known_chain is not None: + chain_id = known_chain['chain_id'] + + if known_chain['is_synced']: + msg = (f'Got invalid InitChain ABCI request ({genesis}) - ' + f'the chain {chain_id} is already synced.') + logger.error(msg) + sys.exit(1) + + if chain_id != genesis.chain_id: + validators = self.bigchaindb.get_validators() + self.log_abci_migration_error(chain_id, validators) + sys.exit(1) + + # set migration values for app hash and height + block = self.bigchaindb.get_latest_block() + app_hash = '' if block is None else block['app_hash'] + height = 0 if block is None else block['height'] + 1 + + known_validators = self.bigchaindb.get_validators() + validator_set = [vutils.decode_validator(v) + for v in genesis.validators] + + if known_validators and known_validators != validator_set: + self.log_abci_migration_error(known_chain['chain_id'], + known_validators) + sys.exit(1) + + block = Block(app_hash=app_hash, height=height, transactions=[]) + self.bigchaindb.store_block(block._asdict()) + self.bigchaindb.store_validator_set(height + 1, validator_set) + abci_chain_height = 0 if known_chain is None else known_chain['height'] + self.bigchaindb.store_abci_chain(abci_chain_height, + genesis.chain_id, True) + self.chain = {'height': abci_chain_height, 'is_synced': True, + 'chain_id': genesis.chain_id} + return self.abci.ResponseInitChain() + + def info(self, request): + """Return height of the latest committed block.""" + + self.abort_if_abci_chain_is_not_synced() + + # Check if Planetmint supports the Tendermint version + if not (hasattr(request, 'version') and tendermint_version_is_compatible(request.version)): + logger.error(f'Unsupported Tendermint version: {getattr(request, "version", "no version")}.' + f' Currently, Planetmint only supports {__tm_supported_versions__}. Exiting!') + sys.exit(1) + + logger.info(f"Tendermint version: {request.version}") + + r = self.abci.ResponseInfo() + block = self.bigchaindb.get_latest_block() + if block: + chain_shift = 0 if self.chain is None else self.chain['height'] + r.last_block_height = block['height'] - chain_shift + r.last_block_app_hash = block['app_hash'].encode('utf-8') + else: + r.last_block_height = 0 + r.last_block_app_hash = b'' + return r + + def check_tx(self, raw_transaction): + """Validate the transaction before entry into + the mempool. + + Args: + raw_tx: a raw string (in bytes) transaction. + """ + + self.abort_if_abci_chain_is_not_synced() + + logger.debug('check_tx: %s', raw_transaction) + transaction = decode_transaction(raw_transaction) + if self.bigchaindb.is_valid_transaction(transaction): + logger.debug('check_tx: VALID') + return self.abci.ResponseCheckTx(code=CodeTypeOk) + else: + logger.debug('check_tx: INVALID') + return self.abci.ResponseCheckTx(code=CodeTypeError) + + def begin_block(self, req_begin_block): + """Initialize list of transaction. + Args: + req_begin_block: block object which contains block header + and block hash. + """ + self.abort_if_abci_chain_is_not_synced() + + chain_shift = 0 if self.chain is None else self.chain['height'] + logger.debug('BEGIN BLOCK, height:%s, num_txs:%s', + req_begin_block.header.height + chain_shift, + req_begin_block.header.num_txs) + + self.block_txn_ids = [] + self.block_transactions = [] + return self.abci.ResponseBeginBlock() + + def deliver_tx(self, raw_transaction): + """Validate the transaction before mutating the state. + + Args: + raw_tx: a raw string (in bytes) transaction. + """ + + self.abort_if_abci_chain_is_not_synced() + + logger.debug('deliver_tx: %s', raw_transaction) + transaction = self.bigchaindb.is_valid_transaction( + decode_transaction(raw_transaction), self.block_transactions) + + if not transaction: + logger.debug('deliver_tx: INVALID') + return self.abci.ResponseDeliverTx(code=CodeTypeError) + else: + logger.debug('storing tx') + self.block_txn_ids.append(transaction.id) + self.block_transactions.append(transaction) + return self.abci.ResponseDeliverTx(code=CodeTypeOk) + + def end_block(self, request_end_block): + """Calculate block hash using transaction ids and previous block + hash to be stored in the next block. + + Args: + height (int): new height of the chain. + """ + + self.abort_if_abci_chain_is_not_synced() + + chain_shift = 0 if self.chain is None else self.chain['height'] + + height = request_end_block.height + chain_shift + self.new_height = height + + # store pre-commit state to recover in case there is a crash during + # `end_block` or `commit` + logger.debug(f'Updating pre-commit state: {self.new_height}') + pre_commit_state = dict(height=self.new_height, + transactions=self.block_txn_ids) + self.bigchaindb.store_pre_commit_state(pre_commit_state) + + block_txn_hash = calculate_hash(self.block_txn_ids) + block = self.bigchaindb.get_latest_block() + + if self.block_txn_ids: + self.block_txn_hash = calculate_hash([block['app_hash'], block_txn_hash]) + else: + self.block_txn_hash = block['app_hash'] + + validator_update = Election.process_block(self.bigchaindb, + self.new_height, + self.block_transactions) + + return self.abci.ResponseEndBlock(validator_updates=validator_update) + + def commit(self): + """Store the new height and along with block hash.""" + + self.abort_if_abci_chain_is_not_synced() + + data = self.block_txn_hash.encode('utf-8') + + # register a new block only when new transactions are received + if self.block_txn_ids: + self.bigchaindb.store_bulk_transactions(self.block_transactions) + + block = Block(app_hash=self.block_txn_hash, + height=self.new_height, + transactions=self.block_txn_ids) + # NOTE: storing the block should be the last operation during commit + # this effects crash recovery. Refer BEP#8 for details + self.bigchaindb.store_block(block._asdict()) + + logger.debug('Commit-ing new block with hash: apphash=%s ,' + 'height=%s, txn ids=%s', data, self.new_height, + self.block_txn_ids) + + if self.events_queue: + event = Event(EventTypes.BLOCK_VALID, { + 'height': self.new_height, + 'transactions': self.block_transactions + }) + self.events_queue.put(event) + + return self.abci.ResponseCommit(data=data) + + +def rollback(b): + pre_commit = b.get_pre_commit_state() + + if pre_commit is None: + # the pre_commit record is first stored in the first `end_block` + return + + latest_block = b.get_latest_block() + if latest_block is None: + logger.error('Found precommit state but no blocks!') + sys.exit(1) + + # NOTE: the pre-commit state is always at most 1 block ahead of the commited state + if latest_block['height'] < pre_commit['height']: + Election.rollback(b, pre_commit['height'], pre_commit['transactions']) + b.delete_transactions(pre_commit['transactions']) diff --git a/bigchaindb/elections/__init__.py b/bigchaindb/elections/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bigchaindb/elections/election.py b/bigchaindb/elections/election.py new file mode 100644 index 0000000..5ed91c7 --- /dev/null +++ b/bigchaindb/elections/election.py @@ -0,0 +1,355 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 +from collections import OrderedDict + +import base58 +from uuid import uuid4 + +from bigchaindb import backend +from bigchaindb.elections.vote import Vote +from bigchaindb.common.exceptions import (InvalidSignature, + MultipleInputsError, + InvalidProposer, + UnequalValidatorSet, + DuplicateTransaction) +from bigchaindb.tendermint_utils import key_from_base64, public_key_to_base64 +from bigchaindb.common.crypto import (public_key_from_ed25519_key) +from bigchaindb.common.transaction import Transaction +from bigchaindb.common.schema import (_validate_schema, + TX_SCHEMA_COMMON, + TX_SCHEMA_CREATE) + + +class Election(Transaction): + """Represents election transactions. + + To implement a custom election, create a class deriving from this one + with OPERATION set to the election operation, ALLOWED_OPERATIONS + set to (OPERATION,), CREATE set to OPERATION. + """ + + OPERATION = None + # Custom validation schema + TX_SCHEMA_CUSTOM = None + # Election Statuses: + ONGOING = 'ongoing' + CONCLUDED = 'concluded' + INCONCLUSIVE = 'inconclusive' + # Vote ratio to approve an election + ELECTION_THRESHOLD = 2 / 3 + + @classmethod + def get_validator_change(cls, bigchain): + """Return the validator set from the most recent approved block + + :return: { + 'height': , + 'validators': + } + """ + latest_block = bigchain.get_latest_block() + if latest_block is None: + return None + return bigchain.get_validator_change(latest_block['height']) + + @classmethod + def get_validators(cls, bigchain, height=None): + """Return a dictionary of validators with key as `public_key` and + value as the `voting_power` + """ + validators = {} + for validator in bigchain.get_validators(height): + # NOTE: we assume that Tendermint encodes public key in base64 + public_key = public_key_from_ed25519_key(key_from_base64(validator['public_key']['value'])) + validators[public_key] = validator['voting_power'] + + return validators + + @classmethod + def recipients(cls, bigchain): + """Convert validator dictionary to a recipient list for `Transaction`""" + + recipients = [] + for public_key, voting_power in cls.get_validators(bigchain).items(): + recipients.append(([public_key], voting_power)) + + return recipients + + @classmethod + def is_same_topology(cls, current_topology, election_topology): + voters = {} + for voter in election_topology: + if len(voter.public_keys) > 1: + return False + + [public_key] = voter.public_keys + voting_power = voter.amount + voters[public_key] = voting_power + + # Check whether the voters and their votes is same to that of the + # validators and their voting power in the network + return current_topology == voters + + def validate(self, bigchain, current_transactions=[]): + """Validate election transaction + + NOTE: + * A valid election is initiated by an existing validator. + + * A valid election is one where voters are validators and votes are + allocated according to the voting power of each validator node. + + Args: + :param bigchain: (Planetmint) an instantiated bigchaindb.lib.BigchainDB object. + :param current_transactions: (list) A list of transactions to be validated along with the election + + Returns: + Election: a Election object or an object of the derived Election subclass. + + Raises: + ValidationError: If the election is invalid + """ + input_conditions = [] + + duplicates = any(txn for txn in current_transactions if txn.id == self.id) + if bigchain.is_committed(self.id) or duplicates: + raise DuplicateTransaction('transaction `{}` already exists' + .format(self.id)) + + if not self.inputs_valid(input_conditions): + raise InvalidSignature('Transaction signature is invalid.') + + current_validators = self.get_validators(bigchain) + + # NOTE: Proposer should be a single node + if len(self.inputs) != 1 or len(self.inputs[0].owners_before) != 1: + raise MultipleInputsError('`tx_signers` must be a list instance of length one') + + # NOTE: Check if the proposer is a validator. + [election_initiator_node_pub_key] = self.inputs[0].owners_before + if election_initiator_node_pub_key not in current_validators.keys(): + raise InvalidProposer('Public key is not a part of the validator set') + + # NOTE: Check if all validators have been assigned votes equal to their voting power + if not self.is_same_topology(current_validators, self.outputs): + raise UnequalValidatorSet('Validator set much be exactly same to the outputs of election') + + return self + + @classmethod + def generate(cls, initiator, voters, election_data, metadata=None): + # Break symmetry in case we need to call an election with the same properties twice + uuid = uuid4() + election_data['seed'] = str(uuid) + + (inputs, outputs) = cls.validate_create(initiator, voters, election_data, metadata) + election = cls(cls.OPERATION, {'data': election_data}, inputs, outputs, metadata) + cls.validate_schema(election.to_dict()) + return election + + @classmethod + def validate_schema(cls, tx): + """Validate the election transaction. Since `ELECTION` extends `CREATE` transaction, all the validations for + `CREATE` transaction should be inherited + """ + _validate_schema(TX_SCHEMA_COMMON, tx) + _validate_schema(TX_SCHEMA_CREATE, tx) + if cls.TX_SCHEMA_CUSTOM: + _validate_schema(cls.TX_SCHEMA_CUSTOM, tx) + + @classmethod + def create(cls, tx_signers, recipients, metadata=None, asset=None): + raise NotImplementedError + + @classmethod + def transfer(cls, tx_signers, recipients, metadata=None, asset=None): + raise NotImplementedError + + @classmethod + def to_public_key(cls, election_id): + return base58.b58encode(bytes.fromhex(election_id)).decode() + + @classmethod + def count_votes(cls, election_pk, transactions, getter=getattr): + votes = 0 + for txn in transactions: + if getter(txn, 'operation') == Vote.OPERATION: + for output in getter(txn, 'outputs'): + # NOTE: We enforce that a valid vote to election id will have only + # election_pk in the output public keys, including any other public key + # along with election_pk will lead to vote being not considered valid. + if len(getter(output, 'public_keys')) == 1 and [election_pk] == getter(output, 'public_keys'): + votes = votes + int(getter(output, 'amount')) + return votes + + def get_commited_votes(self, bigchain, election_pk=None): + if election_pk is None: + election_pk = self.to_public_key(self.id) + txns = list(backend.query.get_asset_tokens_for_public_key(bigchain.connection, + self.id, + election_pk)) + return self.count_votes(election_pk, txns, dict.get) + + def has_concluded(self, bigchain, current_votes=[]): + """Check if the election can be concluded or not. + + * Elections can only be concluded if the validator set has not changed + since the election was initiated. + * Elections can be concluded only if the current votes form a supermajority. + + Custom elections may override this function and introduce additional checks. + """ + if self.has_validator_set_changed(bigchain): + return False + + election_pk = self.to_public_key(self.id) + votes_committed = self.get_commited_votes(bigchain, election_pk) + votes_current = self.count_votes(election_pk, current_votes) + + total_votes = sum(output.amount for output in self.outputs) + if (votes_committed < (2/3) * total_votes) and \ + (votes_committed + votes_current >= (2/3)*total_votes): + return True + + return False + + def get_status(self, bigchain): + election = self.get_election(self.id, bigchain) + if election and election['is_concluded']: + return self.CONCLUDED + + return self.INCONCLUSIVE if self.has_validator_set_changed(bigchain) else self.ONGOING + + def has_validator_set_changed(self, bigchain): + latest_change = self.get_validator_change(bigchain) + if latest_change is None: + return False + + latest_change_height = latest_change['height'] + + election = self.get_election(self.id, bigchain) + + return latest_change_height > election['height'] + + def get_election(self, election_id, bigchain): + return bigchain.get_election(election_id) + + def store(self, bigchain, height, is_concluded): + bigchain.store_election(self.id, height, is_concluded) + + def show_election(self, bigchain): + data = self.asset['data'] + if 'public_key' in data.keys(): + data['public_key'] = public_key_to_base64(data['public_key']['value']) + response = '' + for k, v in data.items(): + if k != 'seed': + response += f'{k}={v}\n' + response += f'status={self.get_status(bigchain)}' + + return response + + @classmethod + def _get_initiated_elections(cls, height, txns): + elections = [] + for tx in txns: + if not isinstance(tx, Election): + continue + + elections.append({'election_id': tx.id, 'height': height, + 'is_concluded': False}) + return elections + + @classmethod + def _get_votes(cls, txns): + elections = OrderedDict() + for tx in txns: + if not isinstance(tx, Vote): + continue + + election_id = tx.asset['id'] + if election_id not in elections: + elections[election_id] = [] + elections[election_id].append(tx) + return elections + + @classmethod + def process_block(cls, bigchain, new_height, txns): + """Looks for election and vote transactions inside the block, records + and processes elections. + + Every election is recorded in the database. + + Every vote has a chance to conclude the corresponding election. When + an election is concluded, the corresponding database record is + marked as such. + + Elections and votes are processed in the order in which they + appear in the block. Elections are concluded in the order of + appearance of their first votes in the block. + + For every election concluded in the block, calls its `on_approval` + method. The returned value of the last `on_approval`, if any, + is a validator set update to be applied in one of the following blocks. + + `on_approval` methods are implemented by elections of particular type. + The method may contain side effects but should be idempotent. To account + for other concluded elections, if it requires so, the method should + rely on the database state. + """ + # elections initiated in this block + initiated_elections = cls._get_initiated_elections(new_height, txns) + + if initiated_elections: + bigchain.store_elections(initiated_elections) + + # elections voted for in this block and their votes + elections = cls._get_votes(txns) + + validator_update = None + for election_id, votes in elections.items(): + election = bigchain.get_transaction(election_id) + if election is None: + continue + + if not election.has_concluded(bigchain, votes): + continue + + validator_update = election.on_approval(bigchain, new_height) + election.store(bigchain, new_height, is_concluded=True) + + return [validator_update] if validator_update else [] + + @classmethod + def rollback(cls, bigchain, new_height, txn_ids): + """Looks for election and vote transactions inside the block and + cleans up the database artifacts possibly created in `process_blocks`. + + Part of the `end_block`/`commit` crash recovery. + """ + + # delete election records for elections initiated at this height and + # elections concluded at this height + bigchain.delete_elections(new_height) + + txns = [bigchain.get_transaction(tx_id) for tx_id in txn_ids] + + elections = cls._get_votes(txns) + for election_id in elections: + election = bigchain.get_transaction(election_id) + election.on_rollback(bigchain, new_height) + + def on_approval(self, bigchain, new_height): + """Override to update the database state according to the + election rules. Consider the current database state to account for + other concluded elections, if required. + """ + raise NotImplementedError + + def on_rollback(self, bigchain, new_height): + """Override to clean up the database artifacts possibly created + in `on_approval`. Part of the `end_block`/`commit` crash recovery. + """ + raise NotImplementedError diff --git a/bigchaindb/elections/vote.py b/bigchaindb/elections/vote.py new file mode 100644 index 0000000..1bc7c56 --- /dev/null +++ b/bigchaindb/elections/vote.py @@ -0,0 +1,64 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from bigchaindb.common.transaction import Transaction +from bigchaindb.common.schema import (_validate_schema, + TX_SCHEMA_COMMON, + TX_SCHEMA_TRANSFER, + TX_SCHEMA_VOTE) + + +class Vote(Transaction): + + OPERATION = 'VOTE' + # NOTE: This class inherits TRANSFER txn type. The `TRANSFER` property is + # overriden to re-use methods from parent class + TRANSFER = OPERATION + ALLOWED_OPERATIONS = (OPERATION,) + # Custom validation schema + TX_SCHEMA_CUSTOM = TX_SCHEMA_VOTE + + def validate(self, bigchain, current_transactions=[]): + """Validate election vote transaction + NOTE: There are no additional validity conditions on casting votes i.e. + a vote is just a valid TRANFER transaction + + For more details refer BEP-21: https://github.com/bigchaindb/BEPs/tree/master/21 + + Args: + bigchain (Planetmint): an instantiated bigchaindb.lib.BigchainDB object. + + Returns: + Vote: a Vote object + + Raises: + ValidationError: If the election vote is invalid + """ + self.validate_transfer_inputs(bigchain, current_transactions) + return self + + @classmethod + def generate(cls, inputs, recipients, election_id, metadata=None): + (inputs, outputs) = cls.validate_transfer(inputs, recipients, election_id, metadata) + election_vote = cls(cls.OPERATION, {'id': election_id}, inputs, outputs, metadata) + cls.validate_schema(election_vote.to_dict()) + return election_vote + + @classmethod + def validate_schema(cls, tx): + """Validate the validator election vote transaction. Since `VOTE` extends `TRANSFER` + transaction, all the validations for `CREATE` transaction should be inherited + """ + _validate_schema(TX_SCHEMA_COMMON, tx) + _validate_schema(TX_SCHEMA_TRANSFER, tx) + _validate_schema(cls.TX_SCHEMA_CUSTOM, tx) + + @classmethod + def create(cls, tx_signers, recipients, metadata=None, asset=None): + raise NotImplementedError + + @classmethod + def transfer(cls, tx_signers, recipients, metadata=None, asset=None): + raise NotImplementedError diff --git a/bigchaindb/events.py b/bigchaindb/events.py new file mode 100644 index 0000000..b3e7f21 --- /dev/null +++ b/bigchaindb/events.py @@ -0,0 +1,109 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from queue import Empty +from collections import defaultdict +from multiprocessing import Queue + + +POISON_PILL = 'POISON_PILL' + + +class EventTypes: + """Container class that holds all the possible + events Planetmint manages. + """ + + # If you add a new Event Type, make sure to add it + # to the docs in docs/server/source/event-plugin-api.rst + ALL = ~0 + BLOCK_VALID = 1 + BLOCK_INVALID = 2 + # NEW_EVENT = 4 + # NEW_EVENT = 8 + # NEW_EVENT = 16... + + +class Event: + """An Event.""" + + def __init__(self, event_type, event_data): + """Creates a new event. + + Args: + event_type (int): the type of the event, see + :class:`~bigchaindb.events.EventTypes` + event_data (obj): the data of the event. + """ + + self.type = event_type + self.data = event_data + + +class Exchange: + """Dispatch events to subscribers.""" + + def __init__(self): + self.publisher_queue = Queue() + self.started_queue = Queue() + + # Map queues> + self.queues = defaultdict(list) + + def get_publisher_queue(self): + """Get the queue used by the publisher. + + Returns: + a :class:`multiprocessing.Queue`. + """ + + return self.publisher_queue + + def get_subscriber_queue(self, event_types=None): + """Create a new queue for a specific combination of event types + and return it. + + Returns: + a :class:`multiprocessing.Queue`. + Raises: + RuntimeError if called after `run` + """ + + try: + self.started_queue.get(timeout=1) + raise RuntimeError('Cannot create a new subscriber queue while Exchange is running.') + except Empty: + pass + + if event_types is None: + event_types = EventTypes.ALL + + queue = Queue() + self.queues[event_types].append(queue) + return queue + + def dispatch(self, event): + """Given an event, send it to all the subscribers. + + Args + event (:class:`~bigchaindb.events.EventTypes`): the event to + dispatch to all the subscribers. + """ + + for event_types, queues in self.queues.items(): + if event.type & event_types: + for queue in queues: + queue.put(event) + + def run(self): + """Start the exchange""" + self.started_queue.put('STARTED') + + while True: + event = self.publisher_queue.get() + if event == POISON_PILL: + return + else: + self.dispatch(event) diff --git a/bigchaindb/exceptions.py b/bigchaindb/exceptions.py new file mode 100644 index 0000000..9e12b7c --- /dev/null +++ b/bigchaindb/exceptions.py @@ -0,0 +1,12 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +class BigchainDBError(Exception): + """Base class for Planetmint exceptions.""" + + +class CriticalDoubleSpend(BigchainDBError): + """Data integrity error that requires attention""" diff --git a/bigchaindb/fastquery.py b/bigchaindb/fastquery.py new file mode 100644 index 0000000..93068f0 --- /dev/null +++ b/bigchaindb/fastquery.py @@ -0,0 +1,50 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from bigchaindb.utils import condition_details_has_owner +from bigchaindb.backend import query +from bigchaindb.common.transaction import TransactionLink + + +class FastQuery(): + """Database queries that join on block results from a single node.""" + + def __init__(self, connection): + self.connection = connection + + def get_outputs_by_public_key(self, public_key): + """Get outputs for a public key""" + txs = list(query.get_owned_ids(self.connection, public_key)) + return [TransactionLink(tx['id'], index) + for tx in txs + for index, output in enumerate(tx['outputs']) + if condition_details_has_owner(output['condition']['details'], + public_key)] + + def filter_spent_outputs(self, outputs): + """Remove outputs that have been spent + + Args: + outputs: list of TransactionLink + """ + links = [o.to_dict() for o in outputs] + txs = list(query.get_spending_transactions(self.connection, links)) + spends = {TransactionLink.from_dict(input_['fulfills']) + for tx in txs + for input_ in tx['inputs']} + return [ff for ff in outputs if ff not in spends] + + def filter_unspent_outputs(self, outputs): + """Remove outputs that have not been spent + + Args: + outputs: list of TransactionLink + """ + links = [o.to_dict() for o in outputs] + txs = list(query.get_spending_transactions(self.connection, links)) + spends = {TransactionLink.from_dict(input_['fulfills']) + for tx in txs + for input_ in tx['inputs']} + return [ff for ff in outputs if ff in spends] diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py new file mode 100644 index 0000000..3033d79 --- /dev/null +++ b/bigchaindb/lib.py @@ -0,0 +1,514 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Module containing main contact points with Tendermint and +MongoDB. + +""" +import logging +from collections import namedtuple +from uuid import uuid4 + +import rapidjson + +try: + from hashlib import sha3_256 +except ImportError: + # NOTE: needed for Python < 3.6 + from sha3 import sha3_256 + +import requests + +import bigchaindb +from bigchaindb import backend, config_utils, fastquery +from bigchaindb.models import Transaction +from bigchaindb.common.exceptions import (SchemaValidationError, + ValidationError, + DoubleSpend) +from bigchaindb.common.transaction_mode_types import (BROADCAST_TX_COMMIT, + BROADCAST_TX_ASYNC, + BROADCAST_TX_SYNC) +from bigchaindb.tendermint_utils import encode_transaction, merkleroot +from bigchaindb import exceptions as core_exceptions +from bigchaindb.validation import BaseValidationRules + + +logger = logging.getLogger(__name__) + + +class BigchainDB(object): + """Bigchain API + + Create, read, sign, write transactions to the database + """ + + def __init__(self, connection=None): + """Initialize the Bigchain instance + + A Bigchain instance has several configuration parameters (e.g. host). + If a parameter value is passed as an argument to the Bigchain + __init__ method, then that is the value it will have. + Otherwise, the parameter value will come from an environment variable. + If that environment variable isn't set, then the value + will come from the local configuration file. And if that variable + isn't in the local configuration file, then the parameter will have + its default value (defined in bigchaindb.__init__). + + Args: + connection (:class:`~bigchaindb.backend.connection.Connection`): + A connection to the database. + """ + config_utils.autoconfigure() + self.mode_commit = BROADCAST_TX_COMMIT + self.mode_list = (BROADCAST_TX_ASYNC, + BROADCAST_TX_SYNC, + self.mode_commit) + self.tendermint_host = bigchaindb.config['tendermint']['host'] + self.tendermint_port = bigchaindb.config['tendermint']['port'] + self.endpoint = 'http://{}:{}/'.format(self.tendermint_host, self.tendermint_port) + + validationPlugin = bigchaindb.config.get('validation_plugin') + + if validationPlugin: + self.validation = config_utils.load_validation_plugin(validationPlugin) + else: + self.validation = BaseValidationRules + + self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) + + def post_transaction(self, transaction, mode): + """Submit a valid transaction to the mempool.""" + if not mode or mode not in self.mode_list: + raise ValidationError('Mode must be one of the following {}.' + .format(', '.join(self.mode_list))) + + tx_dict = transaction.tx_dict if transaction.tx_dict else transaction.to_dict() + payload = { + 'method': mode, + 'jsonrpc': '2.0', + 'params': [encode_transaction(tx_dict)], + 'id': str(uuid4()) + } + # TODO: handle connection errors! + return requests.post(self.endpoint, json=payload) + + def write_transaction(self, transaction, mode): + # This method offers backward compatibility with the Web API. + """Submit a valid transaction to the mempool.""" + response = self.post_transaction(transaction, mode) + return self._process_post_response(response.json(), mode) + + def _process_post_response(self, response, mode): + logger.debug(response) + + error = response.get('error') + if error: + status_code = 500 + message = error.get('message', 'Internal Error') + data = error.get('data', '') + + if 'Tx already exists in cache' in data: + status_code = 400 + + return (status_code, message + ' - ' + data) + + result = response['result'] + if mode == self.mode_commit: + check_tx_code = result.get('check_tx', {}).get('code', 0) + deliver_tx_code = result.get('deliver_tx', {}).get('code', 0) + error_code = check_tx_code or deliver_tx_code + else: + error_code = result.get('code', 0) + + if error_code: + return (500, 'Transaction validation failed') + + return (202, '') + + def store_bulk_transactions(self, transactions): + txns = [] + assets = [] + txn_metadatas = [] + for t in transactions: + transaction = t.tx_dict if t.tx_dict else rapidjson.loads(rapidjson.dumps(t.to_dict())) + if transaction['operation'] == t.CREATE: + asset = transaction.pop('asset') + asset['id'] = transaction['id'] + assets.append(asset) + + metadata = transaction.pop('metadata') + txn_metadatas.append({'id': transaction['id'], + 'metadata': metadata}) + txns.append(transaction) + + backend.query.store_metadatas(self.connection, txn_metadatas) + if assets: + backend.query.store_assets(self.connection, assets) + return backend.query.store_transactions(self.connection, txns) + + def delete_transactions(self, txs): + return backend.query.delete_transactions(self.connection, txs) + + def update_utxoset(self, transaction): + """Update the UTXO set given ``transaction``. That is, remove + the outputs that the given ``transaction`` spends, and add the + outputs that the given ``transaction`` creates. + + Args: + transaction (:obj:`~bigchaindb.models.Transaction`): A new + transaction incoming into the system for which the UTXO + set needs to be updated. + """ + spent_outputs = [ + spent_output for spent_output in transaction.spent_outputs + ] + if spent_outputs: + self.delete_unspent_outputs(*spent_outputs) + self.store_unspent_outputs( + *[utxo._asdict() for utxo in transaction.unspent_outputs] + ) + + def store_unspent_outputs(self, *unspent_outputs): + """Store the given ``unspent_outputs`` (utxos). + + Args: + *unspent_outputs (:obj:`tuple` of :obj:`dict`): Variable + length tuple or list of unspent outputs. + """ + if unspent_outputs: + return backend.query.store_unspent_outputs( + self.connection, *unspent_outputs) + + def get_utxoset_merkle_root(self): + """Returns the merkle root of the utxoset. This implies that + the utxoset is first put into a merkle tree. + + For now, the merkle tree and its root will be computed each + time. This obviously is not efficient and a better approach + that limits the repetition of the same computation when + unnecesary should be sought. For instance, future optimizations + could simply re-compute the branches of the tree that were + affected by a change. + + The transaction hash (id) and output index should be sufficient + to uniquely identify a utxo, and consequently only that + information from a utxo record is needed to compute the merkle + root. Hence, each node of the merkle tree should contain the + tuple (txid, output_index). + + .. important:: The leaves of the tree will need to be sorted in + some kind of lexicographical order. + + Returns: + str: Merkle root in hexadecimal form. + """ + utxoset = backend.query.get_unspent_outputs(self.connection) + # TODO Once ready, use the already pre-computed utxo_hash field. + # See common/transactions.py for details. + hashes = [ + sha3_256( + '{}{}'.format(utxo['transaction_id'], utxo['output_index']).encode() + ).digest() for utxo in utxoset + ] + # TODO Notice the sorted call! + return merkleroot(sorted(hashes)) + + def get_unspent_outputs(self): + """Get the utxoset. + + Returns: + generator of unspent_outputs. + """ + cursor = backend.query.get_unspent_outputs(self.connection) + return (record for record in cursor) + + def delete_unspent_outputs(self, *unspent_outputs): + """Deletes the given ``unspent_outputs`` (utxos). + + Args: + *unspent_outputs (:obj:`tuple` of :obj:`dict`): Variable + length tuple or list of unspent outputs. + """ + if unspent_outputs: + return backend.query.delete_unspent_outputs( + self.connection, *unspent_outputs) + + def is_committed(self, transaction_id): + transaction = backend.query.get_transaction(self.connection, transaction_id) + return bool(transaction) + + def get_transaction(self, transaction_id): + transaction = backend.query.get_transaction(self.connection, transaction_id) + + if transaction: + asset = backend.query.get_asset(self.connection, transaction_id) + metadata = backend.query.get_metadata(self.connection, [transaction_id]) + if asset: + transaction['asset'] = asset + + if 'metadata' not in transaction: + metadata = metadata[0] if metadata else None + if metadata: + metadata = metadata.get('metadata') + + transaction.update({'metadata': metadata}) + + transaction = Transaction.from_dict(transaction) + + return transaction + + def get_transactions(self, txn_ids): + return backend.query.get_transactions(self.connection, txn_ids) + + def get_transactions_filtered(self, asset_id, operation=None, last_tx=None): + """Get a list of transactions filtered on some criteria + """ + txids = backend.query.get_txids_filtered(self.connection, asset_id, + operation, last_tx) + for txid in txids: + yield self.get_transaction(txid) + + def get_outputs_filtered(self, owner, spent=None): + """Get a list of output links filtered on some criteria + + Args: + owner (str): base58 encoded public_key. + spent (bool): If ``True`` return only the spent outputs. If + ``False`` return only unspent outputs. If spent is + not specified (``None``) return all outputs. + + Returns: + :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s + pointing to another transaction's condition + """ + outputs = self.fastquery.get_outputs_by_public_key(owner) + if spent is None: + return outputs + elif spent is True: + return self.fastquery.filter_unspent_outputs(outputs) + elif spent is False: + return self.fastquery.filter_spent_outputs(outputs) + + def get_spent(self, txid, output, current_transactions=[]): + transactions = backend.query.get_spent(self.connection, txid, + output) + transactions = list(transactions) if transactions else [] + if len(transactions) > 1: + raise core_exceptions.CriticalDoubleSpend( + '`{}` was spent more than once. There is a problem' + ' with the chain'.format(txid)) + + current_spent_transactions = [] + for ctxn in current_transactions: + for ctxn_input in ctxn.inputs: + if ctxn_input.fulfills and\ + ctxn_input.fulfills.txid == txid and\ + ctxn_input.fulfills.output == output: + current_spent_transactions.append(ctxn) + + transaction = None + if len(transactions) + len(current_spent_transactions) > 1: + raise DoubleSpend('tx "{}" spends inputs twice'.format(txid)) + elif transactions: + transaction = Transaction.from_db(self, transactions[0]) + elif current_spent_transactions: + transaction = current_spent_transactions[0] + + return transaction + + def store_block(self, block): + """Create a new block.""" + + return backend.query.store_block(self.connection, block) + + def get_latest_block(self): + """Get the block with largest height.""" + + return backend.query.get_latest_block(self.connection) + + def get_block(self, block_id): + """Get the block with the specified `block_id`. + + Returns the block corresponding to `block_id` or None if no match is + found. + + Args: + block_id (int): block id of the block to get. + """ + + block = backend.query.get_block(self.connection, block_id) + latest_block = self.get_latest_block() + latest_block_height = latest_block['height'] if latest_block else 0 + + if not block and block_id > latest_block_height: + return + + result = {'height': block_id, + 'transactions': []} + + if block: + transactions = backend.query.get_transactions(self.connection, block['transactions']) + result['transactions'] = [t.to_dict() for t in Transaction.from_db(self, transactions)] + + return result + + def get_block_containing_tx(self, txid): + """Retrieve the list of blocks (block ids) containing a + transaction with transaction id `txid` + + Args: + txid (str): transaction id of the transaction to query + + Returns: + Block id list (list(int)) + """ + blocks = list(backend.query.get_block_with_transaction(self.connection, txid)) + if len(blocks) > 1: + logger.critical('Transaction id %s exists in multiple blocks', txid) + + return [block['height'] for block in blocks] + + def validate_transaction(self, tx, current_transactions=[]): + """Validate a transaction against the current status of the database.""" + + transaction = tx + + # CLEANUP: The conditional below checks for transaction in dict format. + # It would be better to only have a single format for the transaction + # throught the code base. + if isinstance(transaction, dict): + try: + transaction = Transaction.from_dict(tx) + except SchemaValidationError as e: + logger.warning('Invalid transaction schema: %s', e.__cause__.message) + return False + except ValidationError as e: + logger.warning('Invalid transaction (%s): %s', type(e).__name__, e) + return False + return transaction.validate(self, current_transactions) + + def is_valid_transaction(self, tx, current_transactions=[]): + # NOTE: the function returns the Transaction object in case + # the transaction is valid + try: + return self.validate_transaction(tx, current_transactions) + except ValidationError as e: + logger.warning('Invalid transaction (%s): %s', type(e).__name__, e) + return False + + def text_search(self, search, *, limit=0, table='assets'): + """Return an iterator of assets that match the text search + + Args: + search (str): Text search string to query the text index + limit (int, optional): Limit the number of returned documents. + + Returns: + iter: An iterator of assets that match the text search. + """ + return backend.query.text_search(self.connection, search, limit=limit, + table=table) + + def get_assets(self, asset_ids): + """Return a list of assets that match the asset_ids + + Args: + asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to + retrieve from the database. + + Returns: + list: The list of assets returned from the database. + """ + return backend.query.get_assets(self.connection, asset_ids) + + def get_metadata(self, txn_ids): + """Return a list of metadata that match the transaction ids (txn_ids) + + Args: + txn_ids (:obj:`list` of :obj:`str`): A list of txn_ids to + retrieve from the database. + + Returns: + list: The list of metadata returned from the database. + """ + return backend.query.get_metadata(self.connection, txn_ids) + + @property + def fastquery(self): + return fastquery.FastQuery(self.connection) + + def get_validator_change(self, height=None): + return backend.query.get_validator_set(self.connection, height) + + def get_validators(self, height=None): + result = self.get_validator_change(height) + return [] if result is None else result['validators'] + + def get_election(self, election_id): + return backend.query.get_election(self.connection, election_id) + + def get_pre_commit_state(self): + return backend.query.get_pre_commit_state(self.connection) + + def store_pre_commit_state(self, state): + return backend.query.store_pre_commit_state(self.connection, state) + + def store_validator_set(self, height, validators): + """Store validator set at a given `height`. + NOTE: If the validator set already exists at that `height` then an + exception will be raised. + """ + return backend.query.store_validator_set(self.connection, {'height': height, + 'validators': validators}) + + def delete_validator_set(self, height): + return backend.query.delete_validator_set(self.connection, height) + + def store_abci_chain(self, height, chain_id, is_synced=True): + return backend.query.store_abci_chain(self.connection, height, + chain_id, is_synced) + + def delete_abci_chain(self, height): + return backend.query.delete_abci_chain(self.connection, height) + + def get_latest_abci_chain(self): + return backend.query.get_latest_abci_chain(self.connection) + + def migrate_abci_chain(self): + """Generate and record a new ABCI chain ID. New blocks are not + accepted until we receive an InitChain ABCI request with + the matching chain ID and validator set. + + Chain ID is generated based on the current chain and height. + `chain-X` => `chain-X-migrated-at-height-5`. + `chain-X-migrated-at-height-5` => `chain-X-migrated-at-height-21`. + + If there is no known chain (we are at genesis), the function returns. + """ + latest_chain = self.get_latest_abci_chain() + if latest_chain is None: + return + + block = self.get_latest_block() + + suffix = '-migrated-at-height-' + chain_id = latest_chain['chain_id'] + block_height_str = str(block['height']) + new_chain_id = chain_id.split(suffix)[0] + suffix + block_height_str + + self.store_abci_chain(block['height'] + 1, new_chain_id, False) + + def store_election(self, election_id, height, is_concluded): + return backend.query.store_election(self.connection, election_id, + height, is_concluded) + + def store_elections(self, elections): + return backend.query.store_elections(self.connection, elections) + + def delete_elections(self, height): + return backend.query.delete_elections(self.connection, height) + + +Block = namedtuple('Block', ('app_hash', 'height', 'transactions')) diff --git a/bigchaindb/log.py b/bigchaindb/log.py new file mode 100644 index 0000000..54755bb --- /dev/null +++ b/bigchaindb/log.py @@ -0,0 +1,130 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import bigchaindb +import logging + +from bigchaindb.common.exceptions import ConfigurationError +from logging.config import dictConfig as set_logging_config +import os + + +DEFAULT_LOG_DIR = os.getcwd() + +DEFAULT_LOGGING_CONFIG = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'console': { + 'class': 'logging.Formatter', + 'format': ('[%(asctime)s] [%(levelname)s] (%(name)s) ' + '%(message)s (%(processName)-10s - pid: %(process)d)'), + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + 'file': { + 'class': 'logging.Formatter', + 'format': ('[%(asctime)s] [%(levelname)s] (%(name)s) ' + '%(message)s (%(processName)-10s - pid: %(process)d)'), + 'datefmt': '%Y-%m-%d %H:%M:%S', + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'console', + 'level': logging.INFO, + }, + 'file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': os.path.join(DEFAULT_LOG_DIR, 'bigchaindb.log'), + 'mode': 'w', + 'maxBytes': 209715200, + 'backupCount': 5, + 'formatter': 'file', + 'level': logging.INFO, + }, + 'errors': { + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': os.path.join(DEFAULT_LOG_DIR, 'bigchaindb-errors.log'), + 'mode': 'w', + 'maxBytes': 209715200, + 'backupCount': 5, + 'formatter': 'file', + 'level': logging.ERROR, + } + }, + 'loggers': {}, + 'root': { + 'level': logging.DEBUG, + 'handlers': ['console', 'file', 'errors'], + }, +} + + +def _normalize_log_level(level): + try: + return level.upper() + except AttributeError as exc: + raise ConfigurationError('Log level must be a string!') from exc + + +def setup_logging(): + """Function to configure log hadlers. + + .. important:: + + Configuration, if needed, should be applied before invoking this + decorator, as starting the subscriber process for logging will + configure the root logger for the child process based on the + state of :obj:`bigchaindb.config` at the moment this decorator + is invoked. + + """ + + logging_configs = DEFAULT_LOGGING_CONFIG + new_logging_configs = bigchaindb.config['log'] + + if 'file' in new_logging_configs: + filename = new_logging_configs['file'] + logging_configs['handlers']['file']['filename'] = filename + + if 'error_file' in new_logging_configs: + error_filename = new_logging_configs['error_file'] + logging_configs['handlers']['errors']['filename'] = error_filename + + if 'level_console' in new_logging_configs: + level = _normalize_log_level(new_logging_configs['level_console']) + logging_configs['handlers']['console']['level'] = level + + if 'level_logfile' in new_logging_configs: + level = _normalize_log_level(new_logging_configs['level_logfile']) + logging_configs['handlers']['file']['level'] = level + + if 'fmt_console' in new_logging_configs: + fmt = new_logging_configs['fmt_console'] + logging_configs['formatters']['console']['format'] = fmt + + if 'fmt_logfile' in new_logging_configs: + fmt = new_logging_configs['fmt_logfile'] + logging_configs['formatters']['file']['format'] = fmt + + if 'datefmt_console' in new_logging_configs: + fmt = new_logging_configs['datefmt_console'] + logging_configs['formatters']['console']['datefmt'] = fmt + + if 'datefmt_logfile' in new_logging_configs: + fmt = new_logging_configs['datefmt_logfile'] + logging_configs['formatters']['file']['datefmt'] = fmt + + log_levels = new_logging_configs.get('granular_levels', {}) + + for logger_name, level in log_levels.items(): + level = _normalize_log_level(level) + try: + logging_configs['loggers'][logger_name]['level'] = level + except KeyError: + logging_configs['loggers'][logger_name] = {'level': level} + + set_logging_config(logging_configs) diff --git a/bigchaindb/migrations/__init__.py b/bigchaindb/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bigchaindb/migrations/chain_migration_election.py b/bigchaindb/migrations/chain_migration_election.py new file mode 100644 index 0000000..119392c --- /dev/null +++ b/bigchaindb/migrations/chain_migration_election.py @@ -0,0 +1,48 @@ +import json + +from bigchaindb.common.schema import TX_SCHEMA_CHAIN_MIGRATION_ELECTION +from bigchaindb.elections.election import Election + + +class ChainMigrationElection(Election): + + OPERATION = 'CHAIN_MIGRATION_ELECTION' + CREATE = OPERATION + ALLOWED_OPERATIONS = (OPERATION,) + TX_SCHEMA_CUSTOM = TX_SCHEMA_CHAIN_MIGRATION_ELECTION + + def has_concluded(self, bigchaindb, *args, **kwargs): + chain = bigchaindb.get_latest_abci_chain() + if chain is not None and not chain['is_synced']: + # do not conclude the migration election if + # there is another migration in progress + return False + + return super().has_concluded(bigchaindb, *args, **kwargs) + + def on_approval(self, bigchain, *args, **kwargs): + bigchain.migrate_abci_chain() + + def show_election(self, bigchain): + output = super().show_election(bigchain) + chain = bigchain.get_latest_abci_chain() + if chain is None or chain['is_synced']: + return output + + output += f'\nchain_id={chain["chain_id"]}' + block = bigchain.get_latest_block() + output += f'\napp_hash={block["app_hash"]}' + validators = [ + { + 'pub_key': { + 'type': 'tendermint/PubKeyEd25519', + 'value': k, + }, + 'power': v, + } for k, v in self.get_validators(bigchain).items() + ] + output += f'\nvalidators={json.dumps(validators, indent=4)}' + return output + + def on_rollback(self, bigchain, new_height): + bigchain.delete_abci_chain(new_height) diff --git a/bigchaindb/models.py b/bigchaindb/models.py new file mode 100644 index 0000000..4a0c30e --- /dev/null +++ b/bigchaindb/models.py @@ -0,0 +1,75 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from bigchaindb.backend.schema import validate_language_key +from bigchaindb.common.exceptions import (InvalidSignature, + DuplicateTransaction) +from bigchaindb.common.schema import validate_transaction_schema +from bigchaindb.common.transaction import Transaction +from bigchaindb.common.utils import (validate_txn_obj, validate_key) + + +class Transaction(Transaction): + ASSET = 'asset' + METADATA = 'metadata' + DATA = 'data' + + def validate(self, bigchain, current_transactions=[]): + """Validate transaction spend + Args: + bigchain (Planetmint): an instantiated bigchaindb.Planetmint object. + Returns: + The transaction (Transaction) if the transaction is valid else it + raises an exception describing the reason why the transaction is + invalid. + Raises: + ValidationError: If the transaction is invalid + """ + input_conditions = [] + + if self.operation == Transaction.CREATE: + duplicates = any(txn for txn in current_transactions if txn.id == self.id) + if bigchain.is_committed(self.id) or duplicates: + raise DuplicateTransaction('transaction `{}` already exists' + .format(self.id)) + + if not self.inputs_valid(input_conditions): + raise InvalidSignature('Transaction signature is invalid.') + + elif self.operation == Transaction.TRANSFER: + self.validate_transfer_inputs(bigchain, current_transactions) + + return self + + @classmethod + def from_dict(cls, tx_body): + return super().from_dict(tx_body, False) + + @classmethod + def validate_schema(cls, tx_body): + validate_transaction_schema(tx_body) + validate_txn_obj(cls.ASSET, tx_body[cls.ASSET], cls.DATA, validate_key) + validate_txn_obj(cls.METADATA, tx_body, cls.METADATA, validate_key) + validate_language_key(tx_body[cls.ASSET], cls.DATA) + validate_language_key(tx_body, cls.METADATA) + + +class FastTransaction: + """A minimal wrapper around a transaction dictionary. This is useful for + when validation is not required but a routine expects something that looks + like a transaction, for example during block creation. + + Note: immutability could also be provided + """ + + def __init__(self, tx_dict): + self.data = tx_dict + + @property + def id(self): + return self.data['id'] + + def to_dict(self): + return self.data diff --git a/bigchaindb/parallel_validation.py b/bigchaindb/parallel_validation.py new file mode 100644 index 0000000..ae45798 --- /dev/null +++ b/bigchaindb/parallel_validation.py @@ -0,0 +1,124 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import multiprocessing as mp +from collections import defaultdict + +from bigchaindb import App, BigchainDB +from bigchaindb.tendermint_utils import decode_transaction +from abci import CodeTypeOk + + +class ParallelValidationApp(App): + def __init__(self, bigchaindb=None, events_queue=None, abci=None): + super().__init__(bigchaindb, events_queue, abci=abci) + self.parallel_validator = ParallelValidator() + self.parallel_validator.start() + + def check_tx(self, raw_transaction): + return self.abci.ResponseCheckTx(code=CodeTypeOk) + + def deliver_tx(self, raw_transaction): + self.parallel_validator.validate(raw_transaction) + return self.abci.ResponseDeliverTx(code=CodeTypeOk) + + def end_block(self, request_end_block): + result = self.parallel_validator.result(timeout=30) + for transaction in result: + if transaction: + self.block_txn_ids.append(transaction.id) + self.block_transactions.append(transaction) + + return super().end_block(request_end_block) + + +RESET = 'reset' +EXIT = 'exit' + + +class ParallelValidator: + def __init__(self, number_of_workers=mp.cpu_count()): + self.number_of_workers = number_of_workers + self.transaction_index = 0 + self.routing_queues = [mp.Queue() for _ in range(self.number_of_workers)] + self.workers = [] + self.results_queue = mp.Queue() + + def start(self): + for routing_queue in self.routing_queues: + worker = ValidationWorker(routing_queue, self.results_queue) + process = mp.Process(target=worker.run) + process.start() + self.workers.append(process) + + def stop(self): + for routing_queue in self.routing_queues: + routing_queue.put(EXIT) + + def validate(self, raw_transaction): + dict_transaction = decode_transaction(raw_transaction) + index = int(dict_transaction['id'], 16) % self.number_of_workers + self.routing_queues[index].put((self.transaction_index, dict_transaction)) + self.transaction_index += 1 + + def result(self, timeout=None): + result_buffer = [None] * self.transaction_index + for _ in range(self.transaction_index): + index, transaction = self.results_queue.get(timeout=timeout) + result_buffer[index] = transaction + self.transaction_index = 0 + for routing_queue in self.routing_queues: + routing_queue.put(RESET) + return result_buffer + + +class ValidationWorker: + """Run validation logic in a loop. This Worker is suitable for a Process + life: no thrills, just a queue to get some values, and a queue to return results. + + Note that a worker is expected to validate multiple transactions in + multiple rounds, and it needs to keep in memory all transactions already + validated, until a new round starts. To trigger a new round of validation, + a ValidationWorker expects a `RESET` message. To exit the infinite loop the + worker is in, it expects an `EXIT` message. + """ + + def __init__(self, in_queue, results_queue): + self.in_queue = in_queue + self.results_queue = results_queue + self.bigchaindb = BigchainDB() + self.reset() + + def reset(self): + # We need a place to store already validated transactions, + # in case of dependant transactions in the same block. + # `validated_transactions` maps an `asset_id` with the list + # of all other transactions sharing the same asset. + self.validated_transactions = defaultdict(list) + + def validate(self, dict_transaction): + try: + asset_id = dict_transaction['asset']['id'] + except KeyError: + asset_id = dict_transaction['id'] + + transaction = self.bigchaindb.is_valid_transaction( + dict_transaction, + self.validated_transactions[asset_id]) + + if transaction: + self.validated_transactions[asset_id].append(transaction) + return transaction + + def run(self): + while True: + message = self.in_queue.get() + if message == RESET: + self.reset() + elif message == EXIT: + return + else: + index, transaction = message + self.results_queue.put((index, self.validate(transaction))) diff --git a/bigchaindb/start.py b/bigchaindb/start.py new file mode 100644 index 0000000..d9b7935 --- /dev/null +++ b/bigchaindb/start.py @@ -0,0 +1,90 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import logging +import setproctitle + +from abci import TmVersion, ABCI + +import bigchaindb +from bigchaindb.lib import BigchainDB +from bigchaindb.core import App +from bigchaindb.parallel_validation import ParallelValidationApp +from bigchaindb.web import server, websocket_server +from bigchaindb.events import Exchange, EventTypes +from bigchaindb.utils import Process + + +logger = logging.getLogger(__name__) + +BANNER = """ +**************************************************************************** +* * +* Planetmint 2.2.2 * +* codename "jumping sloth" * +* Initialization complete. Planetmint Server is ready and waiting. * +* * +* You can send HTTP requests via the HTTP API documented in the * +* Planetmint Server docs at: * +* https://bigchaindb.com/http-api * +* * +* Listening to client connections on: {:<15} * +* * +**************************************************************************** +""" + + +def start(args): + # Exchange object for event stream api + logger.info('Starting Planetmint') + exchange = Exchange() + # start the web api + app_server = server.create_server( + settings=bigchaindb.config['server'], + log_config=bigchaindb.config['log'], + bigchaindb_factory=BigchainDB) + p_webapi = Process(name='bigchaindb_webapi', target=app_server.run, daemon=True) + p_webapi.start() + + logger.info(BANNER.format(bigchaindb.config['server']['bind'])) + + # start websocket server + p_websocket_server = Process(name='bigchaindb_ws', + target=websocket_server.start, + daemon=True, + args=(exchange.get_subscriber_queue(EventTypes.BLOCK_VALID),)) + p_websocket_server.start() + + p_exchange = Process(name='bigchaindb_exchange', target=exchange.run, daemon=True) + p_exchange.start() + + # We need to import this after spawning the web server + # because import ABCIServer will monkeypatch all sockets + # for gevent. + from abci.server import ABCIServer + + setproctitle.setproctitle('bigchaindb') + + # Start the ABCIServer + abci = ABCI(TmVersion(bigchaindb.config['tendermint']['version'])) + if args.experimental_parallel_validation: + app = ABCIServer( + app=ParallelValidationApp( + abci=abci.types, + events_queue=exchange.get_publisher_queue(), + ) + ) + else: + app = ABCIServer( + app=App( + abci=abci.types, + events_queue=exchange.get_publisher_queue(), + ) + ) + app.run() + + +if __name__ == '__main__': + start() diff --git a/bigchaindb/tendermint_utils.py b/bigchaindb/tendermint_utils.py new file mode 100644 index 0000000..f613b63 --- /dev/null +++ b/bigchaindb/tendermint_utils.py @@ -0,0 +1,100 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import base64 +import hashlib +import json +from binascii import hexlify + +try: + from hashlib import sha3_256 +except ImportError: + from sha3 import sha3_256 + + +def encode_transaction(value): + """Encode a transaction (dict) to Base64.""" + + return base64.b64encode(json.dumps(value).encode('utf8')).decode('utf8') + + +def decode_transaction(raw): + """Decode a transaction from bytes to a dict.""" + + return json.loads(raw.decode('utf8')) + + +def decode_transaction_base64(value): + """Decode a transaction from Base64.""" + + return json.loads(base64.b64decode(value.encode('utf8')).decode('utf8')) + + +def calculate_hash(key_list): + if not key_list: + return '' + + full_hash = sha3_256() + for key in key_list: + full_hash.update(key.encode('utf8')) + + return full_hash.hexdigest() + + +def merkleroot(hashes): + """Computes the merkle root for a given list. + + Args: + hashes (:obj:`list` of :obj:`bytes`): The leaves of the tree. + + Returns: + str: Merkle root in hexadecimal form. + + """ + # XXX TEMPORARY -- MUST REVIEW and possibly CHANGE + # The idea here is that the UTXO SET would be empty and this function + # would be invoked to compute the merkle root, and since there is nothing, + # i.e. an empty list, then the hash of the empty string is returned. + # This seems too easy but maybe that is good enough? TO REVIEW! + if not hashes: + return sha3_256(b'').hexdigest() + # XXX END TEMPORARY -- MUST REVIEW ... + if len(hashes) == 1: + return hexlify(hashes[0]).decode() + if len(hashes) % 2 == 1: + hashes.append(hashes[-1]) + parent_hashes = [ + sha3_256(hashes[i] + hashes[i+1]).digest() + for i in range(0, len(hashes)-1, 2) + ] + return merkleroot(parent_hashes) + + +def public_key64_to_address(base64_public_key): + """Note this only compatible with Tendermint 0.19.x""" + ed25519_public_key = public_key_from_base64(base64_public_key) + encoded_public_key = amino_encoded_public_key(ed25519_public_key) + return hashlib.new('ripemd160', encoded_public_key).hexdigest().upper() + + +def public_key_from_base64(base64_public_key): + return key_from_base64(base64_public_key) + + +def key_from_base64(base64_key): + return base64.b64decode(base64_key).hex().upper() + + +def public_key_to_base64(ed25519_public_key): + return key_to_base64(ed25519_public_key) + + +def key_to_base64(ed25519_key): + ed25519_key = bytes.fromhex(ed25519_key) + return base64.b64encode(ed25519_key).decode('utf-8') + + +def amino_encoded_public_key(ed25519_public_key): + return bytes.fromhex('1624DE6220{}'.format(ed25519_public_key)) diff --git a/bigchaindb/upsert_validator/__init__.py b/bigchaindb/upsert_validator/__init__.py new file mode 100644 index 0000000..903bb6b --- /dev/null +++ b/bigchaindb/upsert_validator/__init__.py @@ -0,0 +1,7 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +from bigchaindb.upsert_validator.validator_election import ValidatorElection # noqa diff --git a/bigchaindb/upsert_validator/validator_election.py b/bigchaindb/upsert_validator/validator_election.py new file mode 100644 index 0000000..703eb89 --- /dev/null +++ b/bigchaindb/upsert_validator/validator_election.py @@ -0,0 +1,68 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from bigchaindb.common.exceptions import InvalidPowerChange +from bigchaindb.elections.election import Election +from bigchaindb.common.schema import TX_SCHEMA_VALIDATOR_ELECTION +from .validator_utils import (new_validator_set, encode_validator, validate_asset_public_key) + + +class ValidatorElection(Election): + + OPERATION = 'VALIDATOR_ELECTION' + # NOTE: this transaction class extends create so the operation inheritence is achieved + # by renaming CREATE to VALIDATOR_ELECTION + CREATE = OPERATION + ALLOWED_OPERATIONS = (OPERATION,) + TX_SCHEMA_CUSTOM = TX_SCHEMA_VALIDATOR_ELECTION + + def validate(self, bigchain, current_transactions=[]): + """For more details refer BEP-21: https://github.com/bigchaindb/BEPs/tree/master/21 + """ + + current_validators = self.get_validators(bigchain) + + super(ValidatorElection, self).validate(bigchain, current_transactions=current_transactions) + + # NOTE: change more than 1/3 of the current power is not allowed + if self.asset['data']['power'] >= (1/3)*sum(current_validators.values()): + raise InvalidPowerChange('`power` change must be less than 1/3 of total power') + + return self + + @classmethod + def validate_schema(cls, tx): + super(ValidatorElection, cls).validate_schema(tx) + validate_asset_public_key(tx['asset']['data']['public_key']) + + def has_concluded(self, bigchain, *args, **kwargs): + latest_block = bigchain.get_latest_block() + if latest_block is not None: + latest_block_height = latest_block['height'] + latest_validator_change = bigchain.get_validator_change()['height'] + + # TODO change to `latest_block_height + 3` when upgrading to Tendermint 0.24.0. + if latest_validator_change == latest_block_height + 2: + # do not conclude the election if there is a change assigned already + return False + + return super().has_concluded(bigchain, *args, **kwargs) + + def on_approval(self, bigchain, new_height): + validator_updates = [self.asset['data']] + curr_validator_set = bigchain.get_validators(new_height) + updated_validator_set = new_validator_set(curr_validator_set, + validator_updates) + + updated_validator_set = [v for v in updated_validator_set + if v['voting_power'] > 0] + + # TODO change to `new_height + 2` when upgrading to Tendermint 0.24.0. + bigchain.store_validator_set(new_height + 1, updated_validator_set) + return encode_validator(self.asset['data']) + + def on_rollback(self, bigchaindb, new_height): + # TODO change to `new_height + 2` when upgrading to Tendermint 0.24.0. + bigchaindb.delete_validator_set(new_height + 1) diff --git a/bigchaindb/upsert_validator/validator_utils.py b/bigchaindb/upsert_validator/validator_utils.py new file mode 100644 index 0000000..30bec43 --- /dev/null +++ b/bigchaindb/upsert_validator/validator_utils.py @@ -0,0 +1,85 @@ +import base64 +import binascii +import codecs + +import bigchaindb +from abci import types_v0_22_8, types_v0_31_5, TmVersion +from bigchaindb.common.exceptions import InvalidPublicKey, BigchainDBError + + +def encode_validator(v): + ed25519_public_key = v['public_key']['value'] + # NOTE: tendermint expects public to be encoded in go-amino format + try: + version = TmVersion(bigchaindb.config["tendermint"]["version"]) + except ValueError: + raise BigchainDBError('Invalid tendermint version, ' + 'check Planetmint configuration file') + + validator_update_t, pubkey_t = { + TmVersion.v0_22_8: (types_v0_22_8.Validator, types_v0_22_8.PubKey), + TmVersion.v0_31_5: (types_v0_31_5.ValidatorUpdate, types_v0_31_5.PubKey) + }[version] + pub_key = pubkey_t(type='ed25519', data=bytes.fromhex(ed25519_public_key)) + + return validator_update_t(pub_key=pub_key, power=v['power']) + + +def decode_validator(v): + return {'public_key': {'type': 'ed25519-base64', + 'value': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')}, + 'voting_power': v.power} + + +def new_validator_set(validators, updates): + validators_dict = {} + for v in validators: + validators_dict[v['public_key']['value']] = v + + updates_dict = {} + for u in updates: + decoder = get_public_key_decoder(u['public_key']) + public_key64 = base64.b64encode(decoder(u['public_key']['value'])).decode('utf-8') + updates_dict[public_key64] = {'public_key': {'type': 'ed25519-base64', + 'value': public_key64}, + 'voting_power': u['power']} + + new_validators_dict = {**validators_dict, **updates_dict} + return list(new_validators_dict.values()) + + +def encode_pk_to_base16(validator): + pk = validator['public_key'] + decoder = get_public_key_decoder(pk) + public_key16 = base64.b16encode(decoder(pk['value'])).decode('utf-8') + + validator['public_key']['value'] = public_key16 + return validator + + +def validate_asset_public_key(pk): + pk_binary = pk['value'].encode('utf-8') + decoder = get_public_key_decoder(pk) + try: + pk_decoded = decoder(pk_binary) + if len(pk_decoded) != 32: + raise InvalidPublicKey('Public key should be of size 32 bytes') + + except binascii.Error: + raise InvalidPublicKey('Invalid `type` specified for public key `value`') + + +def get_public_key_decoder(pk): + encoding = pk['type'] + decoder = base64.b64decode + + if encoding == 'ed25519-base16': + decoder = base64.b16decode + elif encoding == 'ed25519-base32': + decoder = base64.b32decode + elif encoding == 'ed25519-base64': + decoder = base64.b64decode + else: + raise InvalidPublicKey('Invalid `type` specified for public key `value`') + + return decoder diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py new file mode 100644 index 0000000..789afed --- /dev/null +++ b/bigchaindb/utils.py @@ -0,0 +1,209 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import contextlib +import threading +import queue +import multiprocessing as mp +import json + +import setproctitle +from packaging import version +from bigchaindb.version import __tm_supported_versions__ +from bigchaindb.tendermint_utils import key_from_base64 +from bigchaindb.common.crypto import key_pair_from_ed25519_key + + +class ProcessGroup(object): + + def __init__(self, concurrency=None, group=None, target=None, name=None, + args=None, kwargs=None, daemon=None): + self.concurrency = concurrency or mp.cpu_count() + self.group = group + self.target = target + self.name = name + self.args = args or () + self.kwargs = kwargs or {} + self.daemon = daemon + self.processes = [] + + def start(self): + for i in range(self.concurrency): + proc = mp.Process(group=self.group, target=self.target, + name=self.name, args=self.args, + kwargs=self.kwargs, daemon=self.daemon) + proc.start() + self.processes.append(proc) + + +class Process(mp.Process): + """Wrapper around multiprocessing.Process that uses + setproctitle to set the name of the process when running + the target task. + """ + + def run(self): + setproctitle.setproctitle(self.name) + super().run() + + +# Inspired by: +# - http://stackoverflow.com/a/24741694/597097 +def pool(builder, size, timeout=None): + """Create a pool that imposes a limit on the number of stored + instances. + + Args: + builder: a function to build an instance. + size: the size of the pool. + timeout(Optional[float]): the seconds to wait before raising + a ``queue.Empty`` exception if no instances are available + within that time. + Raises: + If ``timeout`` is defined but the request is taking longer + than the specified time, the context manager will raise + a ``queue.Empty`` exception. + + Returns: + A context manager that can be used with the ``with`` + statement. + + """ + + lock = threading.Lock() + local_pool = queue.Queue() + current_size = 0 + + @contextlib.contextmanager + def pooled(): + nonlocal current_size + instance = None + + # If we still have free slots, then we have room to create new + # instances. + if current_size < size: + with lock: + # We need to check again if we have slots available, since + # the situation might be different after acquiring the lock + if current_size < size: + current_size += 1 + instance = builder() + + # Watchout: current_size can be equal to size if the previous part of + # the function has been executed, that's why we need to check if the + # instance is None. + if instance is None: + instance = local_pool.get(timeout=timeout) + + yield instance + + local_pool.put(instance) + + return pooled + + +# TODO: Rename this function, it's handling fulfillments not conditions +def condition_details_has_owner(condition_details, owner): + """Check if the public_key of owner is in the condition details + as an Ed25519Fulfillment.public_key + + Args: + condition_details (dict): dict with condition details + owner (str): base58 public key of owner + + Returns: + bool: True if the public key is found in the condition details, False otherwise + + """ + if 'subconditions' in condition_details: + result = condition_details_has_owner(condition_details['subconditions'], owner) + if result: + return True + + elif isinstance(condition_details, list): + for subcondition in condition_details: + result = condition_details_has_owner(subcondition, owner) + if result: + return True + else: + if 'public_key' in condition_details \ + and owner == condition_details['public_key']: + return True + return False + + +class Lazy: + """Lazy objects are useful to create chains of methods to + execute later. + + A lazy object records the methods that has been called, and + replay them when the :py:meth:`run` method is called. Note that + :py:meth:`run` needs an object `instance` to replay all the + methods that have been recorded. + """ + + def __init__(self): + """Instantiate a new Lazy object.""" + self.stack = [] + + def __getattr__(self, name): + self.stack.append(name) + return self + + def __call__(self, *args, **kwargs): + self.stack.append((args, kwargs)) + return self + + def __getitem__(self, key): + self.stack.append('__getitem__') + self.stack.append(([key], {})) + return self + + def run(self, instance): + """Run the recorded chain of methods on `instance`. + + Args: + instance: an object. + """ + + last = instance + + for item in self.stack: + if isinstance(item, str): + last = getattr(last, item) + else: + last = last(*item[0], **item[1]) + + self.stack = [] + return last + + +# Load Tendermint's public and private key from the file path +def load_node_key(path): + with open(path) as json_data: + priv_validator = json.load(json_data) + priv_key = priv_validator['priv_key']['value'] + hex_private_key = key_from_base64(priv_key) + return key_pair_from_ed25519_key(hex_private_key) + + +def tendermint_version_is_compatible(running_tm_ver): + """ + Check Tendermint compatability with Planetmint server + + :param running_tm_ver: Version number of the connected Tendermint instance + :type running_tm_ver: str + :return: True/False depending on the compatability with Planetmint server + :rtype: bool + """ + + # Splitting because version can look like this e.g. 0.22.8-40d6dc2e + tm_ver = running_tm_ver.split('-') + if not tm_ver: + return False + for ver in __tm_supported_versions__: + if version.parse(ver) == version.parse(tm_ver[0]): + return True + return False diff --git a/bigchaindb/validation.py b/bigchaindb/validation.py new file mode 100644 index 0000000..f5ed2db --- /dev/null +++ b/bigchaindb/validation.py @@ -0,0 +1,25 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +class BaseValidationRules(): + """Base validation rules for Planetmint. + + A validation plugin must expose a class inheriting from this one via an entry_point. + + All methods listed below must be implemented. + """ + + @staticmethod + def validate_transaction(bigchaindb, transaction): + """See :meth:`bigchaindb.models.Transaction.validate` + for documentation. + """ + return transaction.validate(bigchaindb) + + @staticmethod + def validate_block(bigchaindb, block): + """See :meth:`bigchaindb.models.Block.validate` for documentation.""" + return block.validate(bigchaindb) diff --git a/bigchaindb/version.py b/bigchaindb/version.py new file mode 100644 index 0000000..d4a311d --- /dev/null +++ b/bigchaindb/version.py @@ -0,0 +1,10 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +__version__ = '2.2.2' +__short_version__ = '2.2' + +# Supported Tendermint versions +__tm_supported_versions__ = ["0.31.5", "0.22.8"] diff --git a/bigchaindb/web/__init__.py b/bigchaindb/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bigchaindb/web/routes.py b/bigchaindb/web/routes.py new file mode 100644 index 0000000..62f0369 --- /dev/null +++ b/bigchaindb/web/routes.py @@ -0,0 +1,48 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""API routes definition""" +from flask_restful import Api +from bigchaindb.web.views import ( + assets, + metadata, + blocks, + info, + transactions as tx, + outputs, + validators, +) + + +def add_routes(app): + """Add the routes to an app""" + for (prefix, routes) in API_SECTIONS: + api = Api(app, prefix=prefix) + for ((pattern, resource, *args), kwargs) in routes: + kwargs.setdefault('strict_slashes', False) + api.add_resource(resource, pattern, *args, **kwargs) + + +def r(*args, **kwargs): + return (args, kwargs) + + +ROUTES_API_V1 = [ + r('/', info.ApiV1Index), + r('assets/', assets.AssetListApi), + r('metadata/', metadata.MetadataApi), + r('blocks/', blocks.BlockApi), + r('blocks/', blocks.BlockListApi), + r('transactions/', tx.TransactionApi), + r('transactions', tx.TransactionListApi), + r('outputs/', outputs.OutputListApi), + r('validators/', validators.ValidatorsApi), +] + + +API_SECTIONS = [ + (None, [r('/', info.RootIndex)]), + ('/api/v1/', ROUTES_API_V1), +] diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py new file mode 100644 index 0000000..c525ccd --- /dev/null +++ b/bigchaindb/web/server.py @@ -0,0 +1,118 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module contains basic functions to instantiate the Planetmint API. + +The application is implemented in Flask and runs using Gunicorn. +""" + +import copy +import multiprocessing + +from flask import Flask +from flask_cors import CORS +import gunicorn.app.base + +from bigchaindb import utils +from bigchaindb import BigchainDB +from bigchaindb.web.routes import add_routes +from bigchaindb.web.strip_content_type_middleware import StripContentTypeMiddleware + + +# TODO: Figure out if we do we need all this boilerplate. +class StandaloneApplication(gunicorn.app.base.BaseApplication): + """Run a **wsgi** app wrapping it in a Gunicorn Base Application. + + Adapted from: + - http://docs.gunicorn.org/en/latest/custom.html + """ + + def __init__(self, app, *, options=None): + """Initialize a new standalone application. + + Args: + app: A wsgi Python application. + options (dict): the configuration. + + """ + self.options = options or {} + self.application = app + super().__init__() + + def load_config(self): + # find a better way to pass this such that + # the custom logger class can access it. + custom_log_config = self.options.get('custom_log_config') + self.cfg.env_orig['custom_log_config'] = custom_log_config + + config = dict((key, value) for key, value in self.options.items() + if key in self.cfg.settings and value is not None) + + config['default_proc_name'] = 'bigchaindb_gunicorn' + for key, value in config.items(): + # not sure if we need the `key.lower` here, will just keep + # keep it for now. + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + +def create_app(*, debug=False, threads=1, bigchaindb_factory=None): + """Return an instance of the Flask application. + + Args: + debug (bool): a flag to activate the debug mode for the app + (default: False). + threads (int): number of threads to use + Return: + an instance of the Flask application. + """ + + if not bigchaindb_factory: + bigchaindb_factory = Planetmint + + app = Flask(__name__) + app.wsgi_app = StripContentTypeMiddleware(app.wsgi_app) + + CORS(app) + + app.debug = debug + + app.config['bigchain_pool'] = utils.pool(bigchaindb_factory, size=threads) + + add_routes(app) + + return app + + +def create_server(settings, log_config=None, bigchaindb_factory=None): + """Wrap and return an application ready to be run. + + Args: + settings (dict): a dictionary containing the settings, more info + here http://docs.gunicorn.org/en/latest/settings.html + + Return: + an initialized instance of the application. + """ + + settings = copy.deepcopy(settings) + + if not settings.get('workers'): + settings['workers'] = (multiprocessing.cpu_count() * 2) + 1 + + if not settings.get('threads'): + # Note: Threading is not recommended currently, as the frontend workload + # is largely CPU bound and parallisation across Python threads makes it + # slower. + settings['threads'] = 1 + + settings['custom_log_config'] = log_config + app = create_app(debug=settings.get('debug', False), + threads=settings['threads'], + bigchaindb_factory=bigchaindb_factory) + standalone = StandaloneApplication(app, options=settings) + return standalone diff --git a/bigchaindb/web/strip_content_type_middleware.py b/bigchaindb/web/strip_content_type_middleware.py new file mode 100644 index 0000000..026b96b --- /dev/null +++ b/bigchaindb/web/strip_content_type_middleware.py @@ -0,0 +1,32 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import logging + +logger = logging.getLogger(__name__) + + +class StripContentTypeMiddleware: + """WSGI middleware to strip Content-Type header for GETs.""" + + def __init__(self, app): + """Create the new middleware. + + Args: + app: a flask application + """ + self.app = app + + def __call__(self, environ, start_response): + """Run the middleware and then call the original WSGI application.""" + + if environ['REQUEST_METHOD'] == 'GET': + try: + del environ['CONTENT_TYPE'] + except KeyError: + pass + else: + logger.debug('Remove header "Content-Type" from GET request') + return self.app(environ, start_response) diff --git a/bigchaindb/web/views/__init__.py b/bigchaindb/web/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bigchaindb/web/views/assets.py b/bigchaindb/web/views/assets.py new file mode 100644 index 0000000..d97e7d0 --- /dev/null +++ b/bigchaindb/web/views/assets.py @@ -0,0 +1,55 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module provides the blueprint for some basic API endpoints. + +For more information please refer to the documentation: http://bigchaindb.com/http-api +""" +import logging + +from flask_restful import reqparse, Resource +from flask import current_app + +from bigchaindb.backend.exceptions import OperationError +from bigchaindb.web.views.base import make_error + +logger = logging.getLogger(__name__) + + +class AssetListApi(Resource): + def get(self): + """API endpoint to perform a text search on the assets. + + Args: + search (str): Text search string to query the text index + limit (int, optional): Limit the number of returned documents. + + Return: + A list of assets that match the query. + """ + parser = reqparse.RequestParser() + parser.add_argument('search', type=str, required=True) + parser.add_argument('limit', type=int) + args = parser.parse_args() + + if not args['search']: + return make_error(400, 'text_search cannot be empty') + if not args['limit']: + # if the limit is not specified do not pass None to `text_search` + del args['limit'] + + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + assets = bigchain.text_search(**args) + + try: + # This only works with MongoDB as the backend + return list(assets) + except OperationError as e: + return make_error( + 400, + '({}): {}'.format(type(e).__name__, e) + ) diff --git a/bigchaindb/web/views/base.py b/bigchaindb/web/views/base.py new file mode 100644 index 0000000..a0153b5 --- /dev/null +++ b/bigchaindb/web/views/base.py @@ -0,0 +1,46 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Common classes and methods for API handlers +""" +import logging + +from flask import jsonify, request + +from bigchaindb import config + + +logger = logging.getLogger(__name__) + + +def make_error(status_code, message=None): + if status_code == 404 and message is None: + message = 'Not found' + + response_content = {'status': status_code, 'message': message} + request_info = {'method': request.method, 'path': request.path} + request_info.update(response_content) + + logger.error('HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s', request_info) + + response = jsonify(response_content) + response.status_code = status_code + return response + + +def base_ws_uri(): + """Base websocket URL that is advertised to external clients. + + Useful when the websocket URL advertised to the clients needs to be + customized (typically when running behind NAT, firewall, etc.) + """ + + config_wsserver = config['wsserver'] + + scheme = config_wsserver['advertised_scheme'] + host = config_wsserver['advertised_host'] + port = config_wsserver['advertised_port'] + + return '{}://{}:{}'.format(scheme, host, port) diff --git a/bigchaindb/web/views/blocks.py b/bigchaindb/web/views/blocks.py new file mode 100644 index 0000000..b2986cf --- /dev/null +++ b/bigchaindb/web/views/blocks.py @@ -0,0 +1,58 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module provides the blueprint for the blocks API endpoints. + +For more information please refer to the documentation: http://bigchaindb.com/http-api +""" +from flask import current_app +from flask_restful import Resource, reqparse + +from bigchaindb.web.views.base import make_error + + +class BlockApi(Resource): + def get(self, block_id): + """API endpoint to get details about a block. + + Args: + block_id (str): the id of the block. + + Return: + A JSON string containing the data about the block. + """ + + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + block = bigchain.get_block(block_id=block_id) + + if not block: + return make_error(404) + + return block + + +class BlockListApi(Resource): + def get(self): + """API endpoint to get the related blocks for a transaction. + + Return: + A ``list`` of ``block_id``s that contain the given transaction. The + list may be filtered when provided a status query parameter: + "valid", "invalid", "undecided". + """ + parser = reqparse.RequestParser() + parser.add_argument('transaction_id', type=str, required=True) + + args = parser.parse_args(strict=True) + tx_id = args['transaction_id'] + + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + blocks = bigchain.get_block_containing_tx(tx_id) + + return blocks diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py new file mode 100644 index 0000000..6bf4439 --- /dev/null +++ b/bigchaindb/web/views/info.py @@ -0,0 +1,57 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""API Index endpoint""" + +import flask +from flask_restful import Resource + +from bigchaindb.web.views.base import base_ws_uri +from bigchaindb import version +from bigchaindb.web.websocket_server import EVENTS_ENDPOINT + + +class RootIndex(Resource): + def get(self): + docs_url = [ + 'https://docs.bigchaindb.com/projects/server/en/v', + version.__version__ + '/' + ] + return flask.jsonify({ + 'api': { + 'v1': get_api_v1_info('/api/v1/') + }, + 'docs': ''.join(docs_url), + 'software': 'Planetmint', + 'version': version.__version__, + }) + + +class ApiV1Index(Resource): + def get(self): + return flask.jsonify(get_api_v1_info('/')) + + +def get_api_v1_info(api_prefix): + """Return a dict with all the information specific for the v1 of the + api. + """ + websocket_root = base_ws_uri() + EVENTS_ENDPOINT + docs_url = [ + 'https://docs.bigchaindb.com/projects/server/en/v', + version.__version__, + '/http-client-server-api.html', + ] + + return { + 'docs': ''.join(docs_url), + 'transactions': '{}transactions/'.format(api_prefix), + 'blocks': '{}blocks/'.format(api_prefix), + 'assets': '{}assets/'.format(api_prefix), + 'outputs': '{}outputs/'.format(api_prefix), + 'streams': websocket_root, + 'metadata': '{}metadata/'.format(api_prefix), + 'validators': '{}validators'.format(api_prefix), + } diff --git a/bigchaindb/web/views/metadata.py b/bigchaindb/web/views/metadata.py new file mode 100644 index 0000000..aeaa6e4 --- /dev/null +++ b/bigchaindb/web/views/metadata.py @@ -0,0 +1,55 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module provides the blueprint for some basic API endpoints. + +For more information please refer to the documentation: http://bigchaindb.com/http-api +""" +import logging + +from flask_restful import reqparse, Resource +from flask import current_app + +from bigchaindb.backend.exceptions import OperationError +from bigchaindb.web.views.base import make_error + +logger = logging.getLogger(__name__) + + +class MetadataApi(Resource): + def get(self): + """API endpoint to perform a text search on transaction metadata. + + Args: + search (str): Text search string to query the text index + limit (int, optional): Limit the number of returned documents. + + Return: + A list of metadata that match the query. + """ + parser = reqparse.RequestParser() + parser.add_argument('search', type=str, required=True) + parser.add_argument('limit', type=int) + args = parser.parse_args() + + if not args['search']: + return make_error(400, 'text_search cannot be empty') + if not args['limit']: + del args['limit'] + + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + args['table'] = 'metadata' + metadata = bigchain.text_search(**args) + + try: + # This only works with MongoDB as the backend + return list(metadata) + except OperationError as e: + return make_error( + 400, + '({}): {}'.format(type(e).__name__, e) + ) diff --git a/bigchaindb/web/views/outputs.py b/bigchaindb/web/views/outputs.py new file mode 100644 index 0000000..f8cd298 --- /dev/null +++ b/bigchaindb/web/views/outputs.py @@ -0,0 +1,31 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from flask import current_app +from flask_restful import reqparse, Resource + +from bigchaindb.web.views import parameters + + +class OutputListApi(Resource): + def get(self): + """API endpoint to retrieve a list of links to transaction + outputs. + + Returns: + A :obj:`list` of :cls:`str` of links to outputs. + """ + parser = reqparse.RequestParser() + parser.add_argument('public_key', type=parameters.valid_ed25519, + required=True) + parser.add_argument('spent', type=parameters.valid_bool) + args = parser.parse_args(strict=True) + + pool = current_app.config['bigchain_pool'] + with pool() as bigchain: + outputs = bigchain.get_outputs_filtered(args['public_key'], + args['spent']) + return [{'transaction_id': output.txid, 'output_index': output.output} + for output in outputs] diff --git a/bigchaindb/web/views/parameters.py b/bigchaindb/web/views/parameters.py new file mode 100644 index 0000000..8013680 --- /dev/null +++ b/bigchaindb/web/views/parameters.py @@ -0,0 +1,51 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import re + +from bigchaindb.common.transaction_mode_types import (BROADCAST_TX_COMMIT, + BROADCAST_TX_ASYNC, + BROADCAST_TX_SYNC) + + +def valid_txid(txid): + if re.match('^[a-fA-F0-9]{64}$', txid): + return txid.lower() + raise ValueError('Invalid hash') + + +def valid_bool(val): + val = val.lower() + if val == 'true': + return True + if val == 'false': + return False + raise ValueError('Boolean value must be "true" or "false" (lowercase)') + + +def valid_ed25519(key): + if (re.match('^[1-9a-zA-Z]{43,44}$', key) and not + re.match('.*[Il0O]', key)): + return key + raise ValueError('Invalid base58 ed25519 key') + + +def valid_operation(op): + op = op.upper() + if op == 'CREATE': + return 'CREATE' + if op == 'TRANSFER': + return 'TRANSFER' + raise ValueError('Operation must be "CREATE" or "TRANSFER"') + + +def valid_mode(mode): + if mode == 'async': + return BROADCAST_TX_ASYNC + if mode == 'sync': + return BROADCAST_TX_SYNC + if mode == 'commit': + return BROADCAST_TX_COMMIT + raise ValueError('Mode must be "async", "sync" or "commit"') diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py new file mode 100644 index 0000000..4a371db --- /dev/null +++ b/bigchaindb/web/views/transactions.py @@ -0,0 +1,108 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module provides the blueprint for some basic API endpoints. + +For more information please refer to the documentation: http://bigchaindb.com/http-api +""" +import logging + +from flask import current_app, request, jsonify +from flask_restful import Resource, reqparse + +from bigchaindb.common.transaction_mode_types import BROADCAST_TX_ASYNC +from bigchaindb.common.exceptions import SchemaValidationError, ValidationError +from bigchaindb.web.views.base import make_error +from bigchaindb.web.views import parameters +from bigchaindb.models import Transaction + + +logger = logging.getLogger(__name__) + + +class TransactionApi(Resource): + def get(self, tx_id): + """API endpoint to get details about a transaction. + + Args: + tx_id (str): the id of the transaction. + + Return: + A JSON string containing the data about the transaction. + """ + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + tx = bigchain.get_transaction(tx_id) + + if not tx: + return make_error(404) + + return tx.to_dict() + + +class TransactionListApi(Resource): + def get(self): + parser = reqparse.RequestParser() + parser.add_argument('operation', type=parameters.valid_operation) + parser.add_argument('asset_id', type=parameters.valid_txid, + required=True) + parser.add_argument('last_tx', type=parameters.valid_bool, + required=False) + args = parser.parse_args() + with current_app.config['bigchain_pool']() as bigchain: + txs = bigchain.get_transactions_filtered(**args) + + return [tx.to_dict() for tx in txs] + + def post(self): + """API endpoint to push transactions to the Federation. + + Return: + A ``dict`` containing the data about the transaction. + """ + parser = reqparse.RequestParser() + parser.add_argument('mode', type=parameters.valid_mode, + default=BROADCAST_TX_ASYNC) + args = parser.parse_args() + mode = str(args['mode']) + + pool = current_app.config['bigchain_pool'] + + # `force` will try to format the body of the POST request even if the + # `content-type` header is not set to `application/json` + tx = request.get_json(force=True) + + try: + tx_obj = Transaction.from_dict(tx) + except SchemaValidationError as e: + return make_error( + 400, + message='Invalid transaction schema: {}'.format( + e.__cause__.message) + ) + except ValidationError as e: + return make_error( + 400, + 'Invalid transaction ({}): {}'.format(type(e).__name__, e) + ) + + with pool() as bigchain: + try: + bigchain.validate_transaction(tx_obj) + except ValidationError as e: + return make_error( + 400, + 'Invalid transaction ({}): {}'.format(type(e).__name__, e) + ) + else: + status_code, message = bigchain.write_transaction(tx_obj, mode) + + if status_code == 202: + response = jsonify(tx) + response.status_code = 202 + return response + else: + return make_error(status_code, message) diff --git a/bigchaindb/web/views/validators.py b/bigchaindb/web/views/validators.py new file mode 100644 index 0000000..7c3fcee --- /dev/null +++ b/bigchaindb/web/views/validators.py @@ -0,0 +1,23 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from flask import current_app +from flask_restful import Resource + + +class ValidatorsApi(Resource): + def get(self): + """API endpoint to get validators set. + + Return: + A JSON string containing the validator set of the current node. + """ + + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + validators = bigchain.get_validators() + + return validators diff --git a/bigchaindb/web/websocket_server.py b/bigchaindb/web/websocket_server.py new file mode 100644 index 0000000..2d8f621 --- /dev/null +++ b/bigchaindb/web/websocket_server.py @@ -0,0 +1,184 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""WebSocket server for the Planetmint Event Stream API.""" + +# NOTE +# +# This module contains some functions and utilities that might belong to other +# modules. For now, I prefer to keep everything in this module. Why? Because +# those functions are needed only here. +# +# When we will extend this part of the project and we find that we need those +# functionalities elsewhere, we can start creating new modules and organizing +# things in a better way. + + +import json +import asyncio +import logging +import threading +from uuid import uuid4 +from concurrent.futures import CancelledError + +import aiohttp +from aiohttp import web + +from bigchaindb import config +from bigchaindb.events import EventTypes + + +logger = logging.getLogger(__name__) +POISON_PILL = 'POISON_PILL' +EVENTS_ENDPOINT = '/api/v1/streams/valid_transactions' + + +def _multiprocessing_to_asyncio(in_queue, out_queue, loop): + """Bridge between a synchronous multiprocessing queue + and an asynchronous asyncio queue. + + Args: + in_queue (multiprocessing.Queue): input queue + out_queue (asyncio.Queue): output queue + """ + + while True: + value = in_queue.get() + loop.call_soon_threadsafe(out_queue.put_nowait, value) + + +def eventify_block(block): + for tx in block['transactions']: + if tx.asset: + asset_id = tx.asset.get('id', tx.id) + else: + asset_id = tx.id + yield {'height': block['height'], + 'asset_id': asset_id, + 'transaction_id': tx.id} + + +class Dispatcher: + """Dispatch events to websockets. + + This class implements a simple publish/subscribe pattern. + """ + + def __init__(self, event_source): + """Create a new instance. + + Args: + event_source: a source of events. Elements in the queue + should be strings. + """ + + self.event_source = event_source + self.subscribers = {} + + def subscribe(self, uuid, websocket): + """Add a websocket to the list of subscribers. + + Args: + uuid (str): a unique identifier for the websocket. + websocket: the websocket to publish information. + """ + + self.subscribers[uuid] = websocket + + def unsubscribe(self, uuid): + """Remove a websocket from the list of subscribers. + + Args: + uuid (str): a unique identifier for the websocket. + """ + + del self.subscribers[uuid] + + async def publish(self): + """Publish new events to the subscribers.""" + + while True: + event = await self.event_source.get() + str_buffer = [] + + if event == POISON_PILL: + return + + if isinstance(event, str): + str_buffer.append(event) + + elif event.type == EventTypes.BLOCK_VALID: + str_buffer = map(json.dumps, eventify_block(event.data)) + + for str_item in str_buffer: + for _, websocket in self.subscribers.items(): + await websocket.send_str(str_item) + + +async def websocket_handler(request): + """Handle a new socket connection.""" + + logger.debug('New websocket connection.') + websocket = web.WebSocketResponse() + await websocket.prepare(request) + uuid = uuid4() + request.app['dispatcher'].subscribe(uuid, websocket) + + while True: + # Consume input buffer + try: + msg = await websocket.receive() + except RuntimeError as e: + logger.debug('Websocket exception: %s', str(e)) + break + except CancelledError: + logger.debug('Websocket closed') + break + if msg.type == aiohttp.WSMsgType.CLOSED: + logger.debug('Websocket closed') + break + elif msg.type == aiohttp.WSMsgType.ERROR: + logger.debug('Websocket exception: %s', websocket.exception()) + break + + request.app['dispatcher'].unsubscribe(uuid) + return websocket + + +def init_app(event_source, *, loop=None): + """Init the application server. + + Return: + An aiohttp application. + """ + + dispatcher = Dispatcher(event_source) + + # Schedule the dispatcher + loop.create_task(dispatcher.publish()) + + app = web.Application(loop=loop) + app['dispatcher'] = dispatcher + app.router.add_get(EVENTS_ENDPOINT, websocket_handler) + return app + + +def start(sync_event_source, loop=None): + """Create and start the WebSocket server.""" + + if not loop: + loop = asyncio.get_event_loop() + + event_source = asyncio.Queue(loop=loop) + + bridge = threading.Thread(target=_multiprocessing_to_asyncio, + args=(sync_event_source, event_source, loop), + daemon=True) + bridge.start() + + app = init_app(event_source, loop=loop) + aiohttp.web.run_app(app, + host=config['wsserver']['host'], + port=config['wsserver']['port']) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..6966641 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,35 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +codecov: + branch: master # the branch to show by default + +coverage: + precision: 2 + round: down + range: "70...100" + + status: + project: + default: + target: auto + if_no_uploads: error + + patch: + default: + target: "80%" + if_no_uploads: error + + ignore: # files and folders that will be removed during processing + - "docs/*" + - "tests/*" + - "bigchaindb/version.py" + - "k8s/*" + +comment: + # @stevepeak (from codecov.io) suggested we change 'suggestions' to 'uncovered' + # in the following line. Thanks Steve! + layout: "header, diff, changes, sunburst, uncovered" + behavior: default diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a5e6ab0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +version: '2.2' + +services: + # Build: docker-compose build -d bigchaindb + # Run: docker-compose run -d bdb + mongodb: + image: mongo:3.6 + ports: + - "27017:27017" + command: mongod + restart: always + bigchaindb: + depends_on: + - mongodb + - tendermint + build: + context: . + dockerfile: Dockerfile-dev + volumes: + - ./bigchaindb:/usr/src/app/bigchaindb + - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./htmlcov:/usr/src/app/htmlcov + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini + environment: + PLANETMINT_DATABASE_BACKEND: localmongodb + PLANETMINT_DATABASE_HOST: mongodb + PLANETMINT_DATABASE_PORT: 27017 + PLANETMINT_SERVER_BIND: 0.0.0.0:9984 + PLANETMINT_WSSERVER_HOST: 0.0.0.0 + PLANETMINT_WSSERVER_ADVERTISED_HOST: bigchaindb + PLANETMINT_TENDERMINT_HOST: tendermint + PLANETMINT_TENDERMINT_PORT: 26657 + ports: + - "9984:9984" + - "9985:9985" + - "26658" + healthcheck: + test: ["CMD", "bash", "-c", "curl http://bigchaindb:9984 && curl http://tendermint:26657/abci_query"] + interval: 3s + timeout: 5s + retries: 3 + command: '.ci/entrypoint.sh' + restart: always + tendermint: + image: tendermint/tendermint:v0.31.5 + # volumes: + # - ./tmdata:/tendermint + entrypoint: '' + ports: + - "26656:26656" + - "26657:26657" + command: sh -c "tendermint init && tendermint node --consensus.create_empty_blocks=false --proxy_app=tcp://bigchaindb:26658" + restart: always + bdb: + image: busybox + depends_on: + bigchaindb: + condition: service_healthy + + + # curl client to check the health of development env + curl-client: + image: appropriate/curl + command: /bin/sh -c "curl -s http://bigchaindb:9984/ > /dev/null && curl -s http://tendermint:26657/ > /dev/null" + + # Planetmint setup to do acceptance testing with Python + python-acceptance: + build: + context: . + dockerfile: ./acceptance/python/Dockerfile + volumes: + - ./acceptance/python/docs:/docs + - ./acceptance/python/src:/src + environment: + - PLANETMINT_ENDPOINT=bigchaindb + + # Build docs only + # docker-compose build bdocs + # docker-compose up -d bdocs + bdocs: + depends_on: + - vdocs + build: + context: . + dockerfile: Dockerfile-dev + args: + backend: localmongodb + volumes: + - .:/usr/src/app/ + command: make -C docs/root html + vdocs: + image: nginx + ports: + - '33333:80' + volumes: + - ./docs/root/build/html:/usr/share/nginx/html diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..15c3175 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,51 @@ + + +- [Documentation on ReadTheDocs](http://bigchaindb.readthedocs.org/) +- [Planetmint Upgrade Guides](upgrade-guides/) + +# The Planetmint Documentation Strategy + +* Include explanatory comments and docstrings in your code. Write [Google style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) with a maximum line width of 119 characters. +* For quick overview and help documents, feel free to create `README.md` or other `X.md` files, written using [GitHub-flavored Markdown](https://help.github.com/categories/writing-on-github/). Markdown files render nicely on GitHub. We might auto-convert some .md files into a format that can be included in the long-form documentation. +* We use [Sphinx](http://www.sphinx-doc.org/en/stable/) to generate the long-form documentation in various formats (e.g. HTML, PDF). +* We also use [Sphinx](http://www.sphinx-doc.org/en/stable/) to generate Python code documentation (from docstrings and possibly other sources). +* We also use Sphinx to document all REST APIs, with the help of [the `httpdomain` extension](https://pythonhosted.org/sphinxcontrib-httpdomain/). + +# How to Generate the HTML Version of the Long-Form Documentation + +If you want to generate the HTML version of the long-form documentation on your local machine, you need to have Sphinx and some Sphinx-contrib packages installed. To do that, go to a subdirectory of `docs` (e.g. `docs/server`) and do: +```bash +pip install -r requirements.txt +``` + +If you're building the *Server* docs (in `docs/server`) then you must also do: +```bash +pip install -e ../../ +``` + +Note: Don't put `-e ../../` in the `requirements.txt` file. That will work locally +but not on ReadTheDocs. + +You can then generate the HTML documentation _in that subdirectory_ by doing: +```bash +make html +``` + +It should tell you where the generated documentation (HTML files) can be found. You can view it in your web browser. +# Building Docs with Docker Compose + +You can also use [Docker Compose](https://docs.docker.com/compose/) to build and host docs. + +```text +$ docker-compose up -d bdocs +``` + +The docs will be hosted on port **33333**, and can be accessed over [localhost](http://localhost:33333), [127.0.0.1](http://127.0.0.1:33333) +OR http:/HOST_IP:33333. + + diff --git a/docs/root/Makefile b/docs/root/Makefile new file mode 100644 index 0000000..597a65f --- /dev/null +++ b/docs/root/Makefile @@ -0,0 +1,225 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = -W +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Planetmint.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Planetmint.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Planetmint" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Planetmint" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/root/generate_http_server_api_documentation.py b/docs/root/generate_http_server_api_documentation.py new file mode 100644 index 0000000..f5cbf25 --- /dev/null +++ b/docs/root/generate_http_server_api_documentation.py @@ -0,0 +1,207 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +""" Script to build http examples for http server api docs """ + +import json +import os +import os.path + +from bigchaindb.common.transaction import Transaction, Input, TransactionLink +from bigchaindb import lib +from bigchaindb.web import server + + +TPLS = {} + + +TPLS['index-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(index)s +""" + +TPLS['api-index-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(api_index)s +""" + +TPLS['get-tx-id-request'] = """\ +GET /api/v1/transactions/%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-id-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(tx)s +""" + + +TPLS['get-tx-by-asset-request'] = """\ +GET /api/v1/transactions?operation=TRANSFER&asset_id=%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-by-asset-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +[%(tx_transfer)s, +%(tx_transfer_last)s] +""" + +TPLS['post-tx-request'] = """\ +POST /api/v1/transactions?mode=async HTTP/1.1 +Host: example.com +Content-Type: application/json + +%(tx)s +""" + + +TPLS['post-tx-response'] = """\ +HTTP/1.1 202 Accepted +Content-Type: application/json + +%(tx)s +""" + + +TPLS['get-block-request'] = """\ +GET /api/v1/blocks/%(blockid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-block-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(block)s +""" + + +TPLS['get-block-txid-request'] = """\ +GET /api/v1/blocks?transaction_id=%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-block-txid-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(block_list)s +""" + + +def main(): + """ Main function """ + + ctx = {} + + def pretty_json(data): + return json.dumps(data, indent=2, sort_keys=True) + + client = server.create_app().test_client() + + host = 'example.com:9984' + + # HTTP Index + res = client.get('/', environ_overrides={'HTTP_HOST': host}) + res_data = json.loads(res.data.decode()) + ctx['index'] = pretty_json(res_data) + + # API index + res = client.get('/api/v1/', environ_overrides={'HTTP_HOST': host}) + ctx['api_index'] = pretty_json(json.loads(res.data.decode())) + + # tx create + privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' + pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' + asset = {'msg': 'Hello Planetmint!'} + tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset, metadata={'sequence': 0}) + tx = tx.sign([privkey]) + ctx['tx'] = pretty_json(tx.to_dict()) + ctx['public_keys'] = tx.outputs[0].public_keys[0] + ctx['txid'] = tx.id + + # tx transfer + privkey_transfer = '3AeWpPdhEZzWLYfkfYHBfMFC2r1f8HEaGS9NtbbKssya' + pubkey_transfer = '3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9' + + cid = 0 + input_ = Input(fulfillment=tx.outputs[cid].fulfillment, + fulfills=TransactionLink(txid=tx.id, output=cid), + owners_before=tx.outputs[cid].public_keys) + tx_transfer = Transaction.transfer([input_], [([pubkey_transfer], 1)], asset_id=tx.id, metadata={'sequence': 1}) + tx_transfer = tx_transfer.sign([privkey]) + ctx['tx_transfer'] = pretty_json(tx_transfer.to_dict()) + ctx['public_keys_transfer'] = tx_transfer.outputs[0].public_keys[0] + ctx['tx_transfer_id'] = tx_transfer.id + + # privkey_transfer_last = 'sG3jWDtdTXUidBJK53ucSTrosktG616U3tQHBk81eQe' + pubkey_transfer_last = '3Af3fhhjU6d9WecEM9Uw5hfom9kNEwE7YuDWdqAUssqm' + + cid = 0 + input_ = Input(fulfillment=tx_transfer.outputs[cid].fulfillment, + fulfills=TransactionLink(txid=tx_transfer.id, output=cid), + owners_before=tx_transfer.outputs[cid].public_keys) + tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)], + asset_id=tx.id, metadata={'sequence': 2}) + tx_transfer_last = tx_transfer_last.sign([privkey_transfer]) + ctx['tx_transfer_last'] = pretty_json(tx_transfer_last.to_dict()) + ctx['tx_transfer_last_id'] = tx_transfer_last.id + ctx['public_keys_transfer_last'] = tx_transfer_last.outputs[0].public_keys[0] + + # block + node_private = "5G2kE1zJAgTajkVSbPAQWo4c2izvtwqaNHYsaNpbbvxX" + node_public = "DngBurxfeNVKZWCEcDnLj1eMPAS7focUZTE5FndFGuHT" + signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" + + app_hash = 'f6e0c49c6d94d6924351f25bb334cf2a99af4206339bf784e741d1a5ab599056' + block = lib.Block(height=1, transactions=[tx.to_dict()], app_hash=app_hash) + block_dict = block._asdict() + block_dict.pop('app_hash') + ctx['block'] = pretty_json(block_dict) + ctx['blockid'] = block.height + + # block status + block_list = [ + block.height + ] + ctx['block_list'] = pretty_json(block_list) + + + base_path = os.path.join(os.path.dirname(__file__), + 'source/installation/api/http-samples') + if not os.path.exists(base_path): + os.makedirs(base_path) + + for name, tpl in TPLS.items(): + path = os.path.join(base_path, name + '.http') + code = tpl % ctx + with open(path, 'w') as handle: + handle.write(code) + + +def setup(*_): + """ Fool sphinx into think it's an extension muahaha """ + main() + + +if __name__ == '__main__': + main() diff --git a/docs/root/make.bat b/docs/root/make.bat new file mode 100644 index 0000000..7f41e02 --- /dev/null +++ b/docs/root/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +set I18NSPHINXOPTS=%SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\bigchaindb.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\bigchaindb.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/root/requirements.txt b/docs/root/requirements.txt new file mode 100644 index 0000000..81df20d --- /dev/null +++ b/docs/root/requirements.txt @@ -0,0 +1,9 @@ +Sphinx~=1.0 +recommonmark>=0.4.0 +sphinx-rtd-theme>=0.1.9 +sphinxcontrib-napoleon>=0.4.4 +sphinxcontrib-httpdomain>=1.5.0 +pyyaml>=4.2b1 +aafigure>=0.6 +packaging~=18.0 +wget diff --git a/docs/root/source/_static/CREATE_and_TRANSFER_example.png b/docs/root/source/_static/CREATE_and_TRANSFER_example.png new file mode 100644 index 0000000..f9ef1ee Binary files /dev/null and b/docs/root/source/_static/CREATE_and_TRANSFER_example.png differ diff --git a/docs/root/source/_static/CREATE_example.png b/docs/root/source/_static/CREATE_example.png new file mode 100644 index 0000000..88a933c Binary files /dev/null and b/docs/root/source/_static/CREATE_example.png differ diff --git a/docs/root/source/_static/schemaDB.png b/docs/root/source/_static/schemaDB.png new file mode 100644 index 0000000..4ed7ad7 Binary files /dev/null and b/docs/root/source/_static/schemaDB.png differ diff --git a/docs/root/source/about-bigchaindb.rst b/docs/root/source/about-bigchaindb.rst new file mode 100644 index 0000000..d72bd64 --- /dev/null +++ b/docs/root/source/about-bigchaindb.rst @@ -0,0 +1,130 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +About Planetmint +---------------- + +Basic Facts +=========== + +#. One can store arbitrary data (including encrypted data) in a Planetmint network, within limits: there’s a maximum transaction size. Every transaction has a ``metadata`` section which can store almost any Unicode string (up to some maximum length). Similarly, every CREATE transaction has an ``asset.data`` section which can store almost any Unicode string. +#. The data stored in certain Planetmint transaction fields must not be encrypted, e.g. public keys and amounts. Planetmint doesn’t offer private transactions akin to Zcoin. +#. Once data has been stored in a Planetmint network, it’s best to assume it can’t be change or deleted. +#. Every node in a Planetmint network has a full copy of all the stored data. +#. Every node in a Planetmint network can read all the stored data. +#. Everyone with full access to a Planetmint node (e.g. the sysadmin of a node) can read all the data stored on that node. +#. Everyone given access to a node via the Planetmint HTTP API can find and read all the data stored by Planetmint. The list of people with access might be quite short. +#. If the connection between an external user and a Planetmint node isn’t encrypted (using HTTPS, for example), then a wiretapper can read all HTTP requests and responses in transit. +#. If someone gets access to plaintext (regardless of where they got it), then they can (in principle) share it with the whole world. One can make it difficult for them to do that, e.g. if it is a lot of data and they only get access inside a secure room where they are searched as they leave the room. + +Planetmint for Asset Registrations & Transfers +============================================== + +Planetmint can store data of any kind, but it's designed to be particularly good for storing asset registrations and transfers: + +* The fundamental thing that one sends to a Planetmint network, to be checked and stored (if valid), is a *transaction*, and there are two kinds: CREATE transactions and TRANSFER transactions. +* A CREATE transaction can be use to register any kind of asset (divisible or indivisible), along with arbitrary metadata. +* An asset can have zero, one, or several owners. +* The owners of an asset can specify (crypto-)conditions which must be satisfied by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a TRANSFER transaction. +* Planetmint verifies that the conditions have been satisfied as part of checking the validity of TRANSFER transactions. (Moreover, anyone can check that they were satisfied.) +* Planetmint prevents double-spending of an asset. +* Validated transactions are immutable. + +.. note:: + + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the section titled **A Note about Owners** in the relevant `Planetmint Transactions Spec `_. + +# Production-Ready? + +Depending on your use case, Planetmint may or may not be production-ready. You should ask your service provider. +If you want to go live (into production) with Planetmint, please consult with your service provider. + +Note: Planetmint has an open source license with a "no warranty" section that is typical of open source licenses. This is standard in the software industry. For example, the Linux kernel is used in production by billions of machines even though its license includes a "no warranty" section. Warranties are usually provided above the level of the software license, by service providers. + +Storing Private Data Off-Chain +============================== + +A system could store data off-chain, e.g. in a third-party database, document store, or content management system (CMS) and it could use Planetmint to: + +- Keep track of who has read permissions (or other permissions) in a third-party system. An example of how this could be done is described below. +- Keep a permanent record of all requests made to the third-party system. +- Store hashes of documents-stored-elsewhere, so that a change in any document can be detected. +- Record all handshake-establishing requests and responses between two off-chain parties (e.g. a Diffie-Hellman key exchange), so as to prove that they established an encrypted tunnel (without giving readers access to that tunnel). There are more details about this idea in `the Privacy Protocols repository `_. + +A simple way to record who has read permission on a particular document would be for the third-party system (“DocPile”) to store a CREATE transaction in a Planetmint network for every document+user pair, to indicate that that user has read permissions for that document. The transaction could be signed by DocPile (or maybe by a document owner, as a variation). The asset data field would contain 1) the unique ID of the user and 2) the unique ID of the document. The one output on the CREATE transaction would only be transferable/spendable by DocPile (or, again, a document owner). + +To revoke the read permission, DocPile could create a TRANSFER transaction, to spend the one output on the original CREATE transaction, with a metadata field to say that the user in question no longer has read permission on that document. + +This can be carried on indefinitely, i.e. another TRANSFER transaction could be created by DocPile to indicate that the user now has read permissions again. + +DocPile can figure out if a given user has read permissions on a given document by reading the last transaction in the CREATE → TRANSFER → TRANSFER → etc. chain for that user+document pair. + +There are other ways to accomplish the same thing. The above is just one example. + +You might have noticed that the above example didn’t treat the “read permission” as an asset owned (controlled) by a user because if the permission asset is given to (transferred to or created by) the user then it cannot be controlled any further (by DocPile) until the user transfers it back to DocPile. Moreover, the user could transfer the asset to someone else, which might be problematic. + +Storing Private Data On-Chain, Encrypted +======================================== + +There are many ways to store private data on-chain, encrypted. Every use case has its own objectives and constraints, and the best solution depends on the use case. `The IPDB consulting team `_ can help you design the best solution for your use case. + +Below we describe some example system setups, using various crypto primitives, to give a sense of what’s possible. + +Please note: + +- Ed25519 keypairs are designed for signing and verifying cryptographic signatures, `not for encrypting and decrypting messages `_. For encryption, you should use keypairs designed for encryption, such as X25519. +- If someone (or some group) publishes how to decrypt some encrypted data on-chain, then anyone with access to that encrypted data will be able to get the plaintext. The data can’t be deleted. +- Encrypted data can’t be indexed or searched by MongoDB. (It can index and search the ciphertext, but that’s not very useful.) One might use homomorphic encryption to index and search encrypted data, but MongoDB doesn’t have any plans to support that any time soon. If there is indexing or keyword search needed, then some fields of the ``asset.data`` or ``metadata`` objects can be left as plain text and the sensitive information can be stored in an encrypted child-object. + +System Example 1 +~~~~~~~~~~~~~~~~ + +Encrypt the data with a symmetric key and store the ciphertext on-chain (in ``metadata`` or ``asset.data``). To communicate the key to a third party, use their public key to encrypt the symmetric key and send them that. They can decrypt the symmetric key with their private key, and then use that symmetric key to decrypt the on-chain ciphertext. + +The reason for using a symmetric key along with public/private keypairs is so the ciphertext only has to be stored once. + +System Example 2 +~~~~~~~~~~~~~~~~ + +This example uses `proxy re-encryption `_: + +#. MegaCorp encrypts some data using its own public key, then stores that encrypted data (ciphertext 1) in a Planetmint network. +#. MegaCorp wants to let others read that encrypted data, but without ever sharing their private key and without having to re-encrypt themselves for every new recipient. Instead, they find a “proxy” named Moxie, to provide proxy re-encryption services. +#. Zorban contacts MegaCorp and asks for permission to read the data. +#. MegaCorp asks Zorban for his public key. +#. MegaCorp generates a “re-encryption key” and sends it to their proxy, Moxie. +#. Moxie (the proxy) uses the re-encryption key to encrypt ciphertext 1, creating ciphertext 2. +#. Moxie sends ciphertext 2 to Zorban (or to MegaCorp who forwards it to Zorban). +#. Zorban uses his private key to decrypt ciphertext 2, getting the original un-encrypted data. + +Note: + +- The proxy only ever sees ciphertext. They never see any un-encrypted data. +- Zorban never got the ability to decrypt ciphertext 1, i.e. the on-chain data. +- There are variations on the above flow. + +System Example 3 +~~~~~~~~~~~~~~~~ + +This example uses `erasure coding `_: + +#. Erasure-code the data into n pieces. +#. Encrypt each of the n pieces with a different encryption key. +#. Store the n encrypted pieces on-chain, e.g. in n separate transactions. +#. Share each of the the n decryption keys with a different party. + +If k < N of the key-holders gets and decrypts k of the pieces, they can reconstruct the original plaintext. Less than k would not be enough. + +System Example 4 +~~~~~~~~~~~~~~~~ + +This setup could be used in an enterprise blockchain scenario where a special node should be able to see parts of the data, but the others should not. + +- The special node generates an X25519 keypair (or similar asymmetric *encryption* keypair). +- A Planetmint end user finds out the X25519 public key (encryption key) of the special node. +- The end user creates a valid Planetmint transaction, with either the asset.data or the metadata (or both) encrypted using the above-mentioned public key. +- This is only done for transactions where the contents of asset.data or metadata don't matter for validation, so all node operators can validate the transaction. +- The special node is able to decrypt the encrypted data, but the other node operators can't, and nor can any other end user. diff --git a/docs/root/source/basic-usage.md b/docs/root/source/basic-usage.md new file mode 100644 index 0000000..62b5078 --- /dev/null +++ b/docs/root/source/basic-usage.md @@ -0,0 +1,131 @@ + + +# Basic usage + +## Transactions in Planetmint + +In Planetmint, _transactions_ are used to register, issue, create or transfer +things (e.g. assets). + +Transactions are the most basic kind of record stored by Planetmint. There are +two kinds: CREATE transactions and TRANSFER transactions. + +You can view the transaction specifications in Github, which describe transaction components and the conditions they have to fulfill in order to be valid. + +[Planetmint Transactions Specs](https://github.com/bigchaindb/BEPs/tree/master/13/) + +### CREATE Transactions + +A CREATE transaction can be used to register, issue, create or otherwise +initiate the history of a single thing (or asset) in Planetmint. For example, +one might register an identity or a creative work. The things are often called +"assets" but they might not be literal assets. + +Planetmint supports divisible assets as of Planetmint Server v0.8.0. +That means you can create/register an asset with an initial number of "shares." +For example, A CREATE transaction could register a truckload of 50 oak trees. +Each share of a divisible asset must be interchangeable with each other share; +the shares must be fungible. + +A CREATE transaction can have one or more outputs. +Each output has an associated amount: the number of shares tied to that output. +For example, if the asset consists of 50 oak trees, +one output might have 35 oak trees for one set of owners, +and the other output might have 15 oak trees for another set of owners. + +Each output also has an associated condition: the condition that must be met +(by a TRANSFER transaction) to transfer/spend the output. +Planetmint supports a variety of conditions. +For details, see +the section titled **Transaction Components: Conditions** +in the relevant +[Planetmint Transactions Spec](https://github.com/bigchaindb/BEPs/tree/master/13/). + +![Example Planetmint CREATE transaction](./_static/CREATE_example.png) + +Above we see a diagram of an example Planetmint CREATE transaction. +It has one output: Pam owns/controls three shares of the asset +and there are no other shares (because there are no other outputs). + +Each output also has a list of all the public keys associated +with the conditions on that output. +Loosely speaking, that list might be interpreted as the list of "owners." +A more accurate word might be fulfillers, signers, controllers, +or transfer-enablers. +See the section titled **A Note about Owners** +in the relevant [Planetmint Transactions Spec](https://github.com/bigchaindb/BEPs/tree/master/13/). + +A CREATE transaction must be signed by all the owners. +(If you're looking for that signature, +it's in the one "fulfillment" of the one input, albeit encoded.) + +### TRANSFER Transactions + +A TRANSFER transaction can transfer/spend one or more outputs +on other transactions (CREATE transactions or other TRANSFER transactions). +Those outputs must all be associated with the same asset; +a TRANSFER transaction can only transfer shares of one asset at a time. + +Each input on a TRANSFER transaction connects to one output +on another transaction. +Each input must satisfy the condition on the output it's trying +to transfer/spend. + +A TRANSFER transaction can have one or more outputs, +just like a CREATE transaction (described above). +The total number of shares coming in on the inputs must equal +the total number of shares going out on the outputs. + +![Example Planetmint transactions](./_static/CREATE_and_TRANSFER_example.png) + +Above we see a diagram of two example Planetmint transactions, +a CREATE transaction and a TRANSFER transaction. +The CREATE transaction is the same as in the earlier diagram. +The TRANSFER transaction spends Pam's output, +so the input on that TRANSFER transaction must contain a valid signature +from Pam (i.e. a valid fulfillment). +The TRANSFER transaction has two outputs: +Jim gets one share, and Pam gets the remaining two shares. + +Terminology: The "Pam, 3" output is called a "spent transaction output" +and the "Jim, 1" and "Pam, 2" outputs are called "unspent transaction outputs" +(UTXOs). + +**Example 1:** Suppose a red car is owned and controlled by Joe. +Suppose the current transfer condition on the car says +that any valid transfer must be signed by Joe. +Joe could build a TRANSFER transaction containing +an input with Joe's signature (to fulfill the current output condition) +plus a new output condition saying that any valid transfer +must be signed by Rae. + +**Example 2:** Someone might construct a TRANSFER transaction +that fulfills the output conditions on four +previously-untransferred assets of the same asset type +e.g. paperclips. The amounts might be 20, 10, 45 and 25, say, +for a total of 100 paperclips. +The TRANSFER transaction would also set up new transfer conditions. +For example, maybe a set of 60 paperclips can only be transferred +if Gertrude signs, and a separate set of 40 paperclips can only be +transferred if both Jack and Kelly sign. +Note how the sum of the incoming paperclips must equal the sum +of the outgoing paperclips (100). + +### Transaction Validity + +When a node is asked to check if a transaction is valid, it checks several +things. We documented those things in a post on *The Planetmint Blog*: +["What is a Valid Transaction in Planetmint?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598) +(Note: That post was about Planetmint Server v1.0.0.) + +### Example Transactions + +There are example Planetmint transactions in +[the HTTP API documentation](./installation/api/http-client-server-api) +and +[the Python Driver documentation](./drivers/index). diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py new file mode 100644 index 0000000..19ced6f --- /dev/null +++ b/docs/root/source/conf.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Planetmint documentation build configuration file, created by +# sphinx-quickstart on Thu Sep 29 11:13:27 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import wget +import datetime +import os +import sys +import inspect + +from os import rename, remove +from recommonmark.parser import CommonMarkParser + + + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +# get version +_version = {} +with open('../../../bigchaindb/version.py') as fp: + exec(fp.read(), _version) + + +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0,parentdir) +#sys.path.insert(0, "/home/myname/pythonfiles") + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +import sphinx_rtd_theme + +extensions = [ + 'sphinx.ext.autosectionlabel', + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'sphinx.ext.napoleon', + 'sphinxcontrib.httpdomain', + 'aafigure.sphinxext', + # Below are actually build steps made to look like sphinx extensions. + # It was the easiest way to get it running with ReadTheDocs. + 'generate_http_server_api_documentation', +] + +try: + remove('contributing/cross-project-policies/code-of-conduct.md') + remove('contributing/cross-project-policies/release-process.md') + remove('contributing/cross-project-policies/python-style-guide.md') +except: + print('done') + +def get_old_new(url, old, new): + filename = wget.download(url) + rename(old, new) + +get_old_new('https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/CODE_OF_CONDUCT.md', + 'CODE_OF_CONDUCT.md', 'contributing/cross-project-policies/code-of-conduct.md') + +get_old_new('https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/RELEASE_PROCESS.md', + 'RELEASE_PROCESS.md', 'contributing/cross-project-policies/release-process.md') + +get_old_new('https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/PYTHON_STYLE_GUIDE.md', + 'PYTHON_STYLE_GUIDE.md', 'contributing/cross-project-policies/python-style-guide.md') + +suppress_warnings = ['misc.highlighting_failure'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# autodoc settings +autodoc_member_order = 'bysource' +autodoc_default_options = { + 'members': None, +} + +source_parsers = { + '.md': CommonMarkParser, +} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ['.rst', '.md'] + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'Planetmint' +now = datetime.datetime.now() +copyright = str(now.year) + ', Planetmint Contributors' +author = 'Planetmint Contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = _version['__short_version__'] +# The full version, including alpha/beta/rc tags. +release = _version['__version__'] +# The full version, including alpha/beta/rc tags. + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = 'Planetmint v0.1' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'BigchainDBdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Planetmint.tex', 'Planetmint Documentation', + 'Planetmint Contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'bigchaindb', 'Planetmint Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Planetmint', 'Planetmint Documentation', + author, 'Planetmint', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False diff --git a/docs/root/source/contributing/cross-project-policies/code-of-conduct.md b/docs/root/source/contributing/cross-project-policies/code-of-conduct.md new file mode 100644 index 0000000..bca9eab --- /dev/null +++ b/docs/root/source/contributing/cross-project-policies/code-of-conduct.md @@ -0,0 +1,57 @@ + + +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute to the project. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, nationality, or species--no picking on Wrigley for being a buffalo! + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Deliberate intimidation +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior directed at yourself or another community member may be +reported by contacting a project maintainer at [contact@bigchaindb.com](mailto:contact@bigchaindb.com). All +complaints will be reviewed and investigated and will result in a response that +is appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/docs/root/source/contributing/cross-project-policies/index.rst b/docs/root/source/contributing/cross-project-policies/index.rst new file mode 100644 index 0000000..696a344 --- /dev/null +++ b/docs/root/source/contributing/cross-project-policies/index.rst @@ -0,0 +1,18 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Policies +======== + + +.. toctree:: + :maxdepth: 1 + + code-of-conduct + python-style-guide + JavaScript Style Guide + release-process + \ No newline at end of file diff --git a/docs/root/source/contributing/cross-project-policies/python-style-guide.md b/docs/root/source/contributing/cross-project-policies/python-style-guide.md new file mode 100644 index 0000000..b84b40b --- /dev/null +++ b/docs/root/source/contributing/cross-project-policies/python-style-guide.md @@ -0,0 +1,97 @@ + + +# Python Style Guide + +This guide starts out with our general Python coding style guidelines and ends with a section on how we write & run (Python) tests. + +## General Python Coding Style Guidelines + +Our starting point is [PEP8](https://www.python.org/dev/peps/pep-0008/), the standard "Style Guide for Python Code." Many Python IDEs will check your code against PEP8. (Note that PEP8 isn't frozen; it actually changes over time, but slowly.) + +Planetmint uses Python 3.5+, so you can ignore all PEP8 guidelines specific to Python 2. + +We use [pre-commit](http://pre-commit.com/) to check some of the rules below before every commit but not everything is realized yet. +The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file. + +### Python Docstrings + +PEP8 says some things about docstrings, but not what to put in them or how to structure them. [PEP257](https://www.python.org/dev/peps/pep-0257/) was one proposal for docstring conventions, but we prefer [Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) instead: they're easier to read and the [napoleon extension](http://www.sphinx-doc.org/en/stable/ext/napoleon.html) for Sphinx lets us turn them into nice-looking documentation. Here are some references on Google-style docstrings: + +* [Google's docs on Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) +* [napoleon's docs include an overview of Google-style docstrings](http://sphinxcontrib-napoleon.readthedocs.org/en/latest/index.html) +* [Example Google-style docstrings](http://sphinxcontrib-napoleon.readthedocs.org/en/latest/example_google.html) (from napoleon's docs) + +### Maximum Line Length + +PEP8 has some [maximum line length guidelines](https://www.python.org/dev/peps/pep-0008/#id17), starting with "Limit all lines to a maximum of 79 characters" but "for flowing long blocks of text with fewer structural restrictions (docstrings or comments), the line length should be limited to 72 characters." + +We discussed this at length, and it seems that the consensus is: _try_ to keep line lengths less than 79/72 characters, unless you have a special situation where longer lines would improve readability. (The basic reason is that 79/72 works for everyone, and Planetmint is an open source project.) As a hard limit, keep all lines less than 119 characters (which is the width of GitHub code review). + +### Single or Double Quotes? + +Python lets you use single or double quotes. PEP8 says you can use either, as long as you're consistent. We try to stick to using single quotes, except in cases where using double quotes is more readable. For example: +```python +print('This doesn\'t look so nice.') +print("Doesn't this look nicer?") +``` + +### Breaking Strings Across Multiple Lines + +Should we use parentheses or slashes (`\`) to break strings across multiple lines, i.e. +```python +my_string = ('This is a very long string, so long that it will not fit into just one line ' + 'so it must be split across multiple lines.') +# or +my_string = 'This is a very long string, so long that it will not fit into just one line ' \ + 'so it must be split across multiple lines.' +``` + +It seems the preference is for slashes, but using parentheses is okay too. (There are good arguments either way. Arguing about it seems like a waste of time.) + +### How to Format Long import Statements + +If you need to `import` lots of names from a module or package, and they won't all fit in one line (without making the line too long), then use parentheses to spread the names across multiple lines, like so: +```python +from Tkinter import ( + Tk, Frame, Button, Entry, Canvas, Text, + LEFT, DISABLED, NORMAL, RIDGE, END, +) + +# Or + +from Tkinter import (Tk, Frame, Button, Entry, Canvas, Text, + LEFT, DISABLED, NORMAL, RIDGE, END) +``` + +For the rationale, see [PEP 328](https://www.python.org/dev/peps/pep-0328/#rationale-for-parentheses). + +### Using the % operator or `format()` to Format Strings + +Given the choice: +```python +x = 'name: %s; score: %d' % (name, n) +# or +x = 'name: {}; score: {}'.format(name, n) +``` + +we use the `format()` version. The [official Python documentation says](https://docs.python.org/2/library/stdtypes.html#str.format), "This method of string formatting is the new standard in Python 3, and should be preferred to the % formatting described in String Formatting Operations in new code." + + +## Running the Flake8 Style Checker + +We use [Flake8](http://flake8.pycqa.org/en/latest/index.html) to check our Python code style. Once you have it installed, you can run it using: +```text +flake8 --max-line-length 119 bigchaindb/ +``` + + +## Writing and Running (Python) Tests + +The content of this section was moved to [`bigchaindb/tests/README.md`](https://github.com/bigchaindb/bigchaindb/blob/master/tests/README.md). + +Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions. diff --git a/docs/root/source/contributing/cross-project-policies/release-process.md b/docs/root/source/contributing/cross-project-policies/release-process.md new file mode 100644 index 0000000..117e407 --- /dev/null +++ b/docs/root/source/contributing/cross-project-policies/release-process.md @@ -0,0 +1,101 @@ + + +# Our Release Process + +## Notes + +Planetmint follows +[the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) +(i.e. MAJOR.MINOR.PATCH), +which is almost identical +to [regular semantic versioning](http://semver.org/), but there's no hyphen, e.g. + +- `0.9.0` for a typical final release +- `4.5.2a1` not `4.5.2-a1` for the first Alpha release +- `3.4.5rc2` not `3.4.5-rc2` for Release Candidate 2 + +**Note 1:** For Git tags (which are used to identify releases on GitHub), we append a `v` in front. For example, the Git tag for version `2.0.0a1` was `v2.0.0a1`. + +**Note 2:** For Docker image tags (e.g. on Docker Hub), we use longer version names for Alpha, Beta and Release Candidate releases. For example, the Docker image tag for version `2.0.0a2` was `2.0.0-alpha2`. + +We use `0.9` and `0.9.0` as example version and short-version values below. You should replace those with the correct values for your new version. + +We follow [BEP-1](https://github.com/bigchaindb/BEPs/tree/master/1), which is our variant of C4, the Collective Code Construction Contract, so a release is just a [tagged commit](https://git-scm.com/book/en/v2/Git-Basics-Tagging) on the `master` branch, i.e. a label for a particular Git commit. + +The following steps are what we do to release a new version of _BigchainDB Server_. The steps to release the Python Driver are similar but not the same. + +## Steps + +1. Create a pull request where you make the following changes: + + - Update `CHANGELOG.md` + - Update all Docker image tags in all Kubernetes YAML files (in the `k8s/` directory). + For example, in the files: + + - `k8s/bigchaindb/bigchaindb-ss.yaml` and + - `k8s/dev-setup/bigchaindb.yaml` + + find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.) + Keep in mind that this is a _Docker image tag_ so our naming convention is + a bit different; see Note 2 in the **Notes** section above. + - In `bigchaindb/version.py`: + - update `__version__` to e.g. `0.9.0` (with no `.dev` on the end) + - update `__short_version__` to e.g. `0.9` (with no `.dev` on the end) + - In the docs about installing Planetmint (and Tendermint), and in the associated scripts, recommend/install a version of Tendermint that _actually works_ with the soon-to-be-released version of Planetmint. You can find all such references by doing a search for the previously-recommended version number, such as `0.31.5`. + - In `setup.py`, _maybe_ update the development status item in the `classifiers` list. For example, one allowed value is `"Development Status :: 5 - Production/Stable"`. The [allowed values are listed at pypi.python.org](https://pypi.python.org/pypi?%3Aaction=list_classifiers). + +2. **Wait for all the tests to pass!** +3. Merge the pull request into the `master` branch. +4. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) + and click the "Draft a new release" button. +5. Fill in the details: + - **Tag version:** version number preceded by `v`, e.g. `v0.9.1` + - **Target:** the last commit that was just merged. In other words, that commit will get a Git tag with the value given for tag version above. + - **Title:** Same as tag version above, e.g `v0.9.1` + - **Description:** The body of the changelog entry (Added, Changed, etc.) +6. Click "Publish release" to publish the release on GitHub. +7. On your local computer, make sure you're on the `master` branch and that it's up-to-date with the `master` branch in the bigchaindb/bigchaindb repository (e.g. `git pull upstream master`). We're going to use that to push a new `bigchaindb` package to PyPI. +8. Make sure you have a `~/.pypirc` file containing credentials for PyPI. +9. Do `make release` to build and publish the new `bigchaindb` package on PyPI. For this step you need to have `twine` installed. If you get an error like `Makefile:135: recipe for target 'clean-pyc' failed` then try doing + ```text + sudo chown -R $(whoami):$(whoami) . + ``` +10. [Log in to readthedocs.org](https://readthedocs.org/accounts/login/) and go to the **Planetmint Server** project, then: + - Click on "Builds", select "latest" from the drop-down menu, then click the "Build Version:" button. + - Wait for the build of "latest" to finish. This can take a few minutes. + - Go to Admin --> Advanced Settings + and make sure that "Default branch:" (i.e. what "latest" points to) + is set to the new release's tag, e.g. `v0.9.1`. + (It won't be an option if you didn't wait for the build of "latest" to finish.) + Then scroll to the bottom and click "Save". + - Go to Admin --> Versions + and under **Choose Active Versions**, do these things: + 1. Make sure that the new version's tag is "Active" and "Public" + 2. Make sure the **stable** branch is _not_ active. + 3. Scroll to the bottom of the page and click "Save". +11. Go to [Docker Hub](https://hub.docker.com/) and sign in, then: + - Click on "Organizations" + - Click on "bigchaindb" + - Click on "bigchaindb/bigchaindb" + - Click on "Build Settings" + - Find the row where "Docker Tag Name" equals `latest` + and change the value of "Name" to the name (Git tag) + of the new release, e.g. `v0.9.0`. + - If the release is an Alpha, Beta or Release Candidate release, + then a new row must be added. + You can do that by clicking the green "+" (plus) icon. + The contents of the new row should be similar to the existing rows + of previous releases like that. + - Click on "Tags" + - Delete the "latest" tag (so we can rebuild it) + - Click on "Build Settings" again + - Click on the "Trigger" button for the "latest" tag and make sure it worked by clicking on "Tags" again + - If the release is an Alpha, Beta or Release Candidate release, + then click on the "Trigger" button for that tag as well. + +Congratulations, you have released a new version of Planetmint Server! diff --git a/docs/root/source/contributing/dev-setup-coding-and-contribution-process/index.rst b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/index.rst new file mode 100644 index 0000000..0e7a8b3 --- /dev/null +++ b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/index.rst @@ -0,0 +1,17 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Developer Setup, Coding & Contribution Process +============================================== + +.. toctree:: + :maxdepth: 2 + + write-code + run-node-with-docker-compose + run-node-as-processes + run-dev-network-stack + run-dev-network-ansible diff --git a/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md new file mode 100644 index 0000000..f98685f --- /dev/null +++ b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md @@ -0,0 +1,170 @@ + + +# Run a Planetmint network with Ansible + +**NOT for Production Use** + +You can use the following instructions to deploy a single or multi node +Planetmint network for dev/test using Ansible. Ansible will configure the Planetmint node(s). + +Currently, this workflow is only supported for the following distributions: +- Ubuntu >= 16.04 +- CentOS >= 7 +- Fedora >= 24 +- MacOSX + +## Minimum Requirements +Minimum resource requirements for a single node Planetmint dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 + +## Clone the Planetmint repository +```text +$ git clone https://github.com/bigchaindb/bigchaindb.git +``` + +## Install dependencies +- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) + +You can also install `ansible` and other dependencies, if any, using the `boostrap.sh` script +inside the Planetmint repository. +Navigate to `bigchaindb/pkg/scripts` and run the `bootstrap.sh` script to install the dependencies +for your OS. The script also checks if the OS you are running is compatible with the +supported versions. + +**Note**: `bootstrap.sh` only supports Ubuntu >= 16.04, CentOS >= 7 and Fedora >=24 and MacOSX. + +```text +$ cd bigchaindb/pkg/scripts/ +$ bash bootstrap.sh --operation install +``` + +### Planetmint Setup Configuration(s) +#### Local Setup +You can run the Ansible playbook `bigchaindb-start.yml` on your local dev machine and set up the Planetmint node where +Planetmint can be run as a process or inside a Docker container(s) depending on your configuration. + +Before, running the playbook locally, you need to update the `hosts` and `stack-config.yml` configuration, which will notify Ansible that we need to run the play locally. + +##### Update Hosts +Navigate to `bigchaindb/pkg/configuration/hosts` inside the Planetmint repository. +```text +$ cd bigchaindb/pkg/configuration/hosts +``` + +Edit `all` configuration file: +```text +# Delete any existing configuration in this file and insert +# Hostname of dev machine + ansible_connection=local +``` +##### Update Configuration +Navigate to `bigchaindb/pkg/configuration/vars` inside the Planetmint repository. +```text +$ cd bigchaindb/pkg/configuration/vars/stack-config.yml +``` + +Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): +```text +--- +stack_type: "docker" +stack_size: "4" + + +OR + +--- +stack_type: "local" +stack_type: "1" +``` + +### Planetmint Setup +Now, You can safely run the `bigchaindb-start.yml` playbook and everything will be taken care of by `Ansible`. To run the playbook please navigate to the `bigchaindb/pkg/configuration` directory inside the Planetmint repository and run the `bigchaindb-start.yml` playbook. + +```text +$ cd bigchaindb/pkg/configuration/ + +$ ansible-playbook bigchaindb-start.yml -i hosts/all --extra-vars "operation=start home_path=$(pwd)" +``` + +After successful execution of the playbook, you can verify that Planetmint docker(s)/process(es) is(are) running. + +Verify Planetmint process(es): +```text +$ ps -ef | grep bigchaindb +``` + +OR + +Verify Planetmint Docker(s): +```text +$ docker ps | grep bigchaindb +``` + +You can now send transactions and verify the functionality of your Planetmint node. +See the [Planetmint Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) +for details on how to use it. + +**Note**: The `bdb_root_url` can be be one of the following: +```text +# Planetmint is running as a process +bdb_root_url = http://:9984 + +OR + +# Planetmint is running inside a docker container +bdb_root_url = http://: +``` + +**Note**: Planetmint has [other drivers as well](http://docs.bigchaindb.com/projects/server/en/latest/drivers-clients/index.html). + +### Experimental: Running Ansible a Remote Dev/Host +#### Remote Setup +You can also run the Ansible playbook `bigchaindb-start.yml` on remote machine(s) and set up the Planetmint node where +Planetmint can run as a process or inside a Docker container(s) depending on your configuration. + +Before, running the playbook on a remote host, you need to update the `hosts` and `stack-config.yml` configuration, which will notify Ansible that we need to run the play on a remote host. + +##### Update Remote Hosts +Navigate to `bigchaindb/pkg/configuration/hosts` inside the Planetmint repository. +```text +$ cd bigchaindb/pkg/configuration/hosts +``` + +Edit `all` configuration file: +```text +# Delete any existing configuration in this file and insert + ansible_ssh_user= ansible_sudo_pass= +``` + +**Note**: You can add multiple hosts to the `all` configuration file. Non-root user with sudo enabled password is needed because ansible will run some tasks that require those permissions. + +**Note**: You can also use other methods to get inside the remote machines instead of password based SSH. For other methods +please consult [Ansible Documentation](http://docs.ansible.com/ansible/latest/intro_getting_started.html). + +##### Update Remote Configuration +Navigate to `bigchaindb/pkg/configuration/vars` inside the Planetmint repository. +```text +$ cd bigchaindb/pkg/configuration/vars/stack-config.yml +``` + +Edit `stack-config.yml` configuration file as per your requirements, sample configuration file(s): +```text +--- +stack_type: "docker" +stack_size: "4" + + +OR + +--- +stack_type: "local" +stack_type: "1" +``` + +After, the configuration of remote hosts, [run the Ansible playbook and verify your deployment](#bigchaindb-setup-ansible). diff --git a/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-dev-network-stack.md b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-dev-network-stack.md new file mode 100644 index 0000000..8b2e7b6 --- /dev/null +++ b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-dev-network-stack.md @@ -0,0 +1,324 @@ + + +# Run a Planetmint network + +**NOT for Production Use** + +You can use the following instructions to deploy a single or multi node +Planetmint network for dev/test using the extensible `stack` script(s). + +Currently, this workflow is only supported for the following Operating systems: +- Ubuntu >= 16.04 +- CentOS >= 7 +- Fedora >= 24 +- MacOSX + +## Machine Minimum Requirements +Minimum resource requirements for a single node Planetmint dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 + +## Download the scripts +> **Note**: If you're working on Planetmint Server code, on a branch based on +> recent code, then you already have local recent versions of *stack.sh* and +> *unstack.sh* in your bigchaindb/pkg/scripts/ directory. Otherwise you can +> get them using: + +```text +$ export GIT_BRANCH=master +$ curl -fOL https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/stack.sh + +# Optional +$ curl -fOL https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/unstack.sh +``` + +## Quick Start +If you run `stack.sh` out of the box i.e. without any configuration changes, you will be able to deploy a 4 node +Planetmint network with Docker containers, created from `master` branch of `bigchaindb/bigchaindb` repo and Tendermint version `0.22.8`. + +**Note**: Run `stack.sh` with either root or non-root user with sudo enabled. + +```text +$ bash stack.sh +...Logs.. +......... +......... +Finished stacking! +``` + +## Configure the Planetmint network + +The `stack.sh` script has multiple deployment methods and parameters and they can be explored using: `bash stack.sh -h` + +```text +$ bash stack.sh -h + + Usage: $ bash stack.sh [-h] + + Deploys the Planetmint network. + + ENV[STACK_SIZE] + Set STACK_SIZE environment variable to the size of the network you desire. + Network mimics a production network environment with single or multiple BDB + nodes. (default: 4). + + ENV[STACK_TYPE] + Set STACK_TYPE environment variable to the type of deployment you desire. + You can set it one of the following: ["docker", "local", "cloud"]. + (default: docker) + + ENV[STACK_TYPE_PROVIDER] + Set only when STACK_TYPE="cloud". Only "azure" is supported. + (default: ) + + ENV[STACK_VM_MEMORY] + (Optional) Set only when STACK_TYPE="local". This sets the memory + of the instance(s) spawned. (default: 2048) + + ENV[STACK_VM_CPUS] + (Optional) Set only when STACK_TYPE="local". This sets the number of VCPUs + of the instance(s) spawned. (default: 2) + + ENV[STACK_BOX_NAME] + (Optional) Set only when STACK_TYPE="local". This sets the box Vagrant box name + of the instance(s) spawned. (default: ubuntu/xenial64) + + ENV[STACK_REPO] + (Optional) To configure bigchaindb repo to use, set STACK_REPO environment + variable. (default: bigchaindb/bigchaindb) + + ENV[STACK_BRANCH] + (Optional) To configure bigchaindb repo branch to use set STACK_BRANCH environment + variable. (default: master) + + ENV[TM_VERSION] + (Optional) Tendermint version to use for the setup. (default: 0.22.8) + + ENV[MONGO_VERSION] + (Optional) MongoDB version to use with the setup. (default: 3.6) + + ENV[AZURE_CLIENT_ID] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_TENANT_ID] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_SUBSCRIPTION_ID] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_CLIENT_SECRET] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_REGION] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Azure region for the Planetmint instance. Get list of regions using Azure CLI. + e.g. az account list-locations. (default: westeurope) + + ENV[AZURE_IMAGE_URN] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Azure image to use. Get list of available images using Azure CLI. + e.g. az vm image list --output table. (default: Canonical:UbuntuServer:16.04-LTS:latest) + + ENV[AZURE_RESOURCE_GROUP] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Name of Azure resource group for the instance. + (default: bdb-vagrant-rg-2018-05-30) + + ENV[AZURE_DNS_PREFIX] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + DNS Prefix of the instance. (default: bdb-instance-2018-05-30) + + ENV[AZURE_ADMIN_USERNAME] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Admin username of the the instance. (default: vagrant) + + ENV[AZURE_VM_SIZE] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Azure VM size. (default: Standard_D2_v2) + + ENV[SSH_PRIVATE_KEY_PATH] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Absolute path of + SSH keypair required to log into the Azure instance. + + -h + Show this help and exit. +``` + + +The parameter that differentiates between the deployment type is `STACK_TYPE` which currently, supports +an opinionated deployment of Planetmint on `docker`, `local` and `cloud`. + +### STACK_TYPE: docker +This configuration deploys a docker based Planetmint network on the dev/test machine that you are running `stack.sh` on. This is also the default `STACK_TYPE` config for `stack.sh`. + +#### Example: docker +Deploy a 4 node docker based Planetmint network on your host. + +```text +#Optional, since 4 is the default size. +$ export STACK_SIZE=4 + +#Optional, since docker is the default type. +$ export STACK_TYPE=docker + +#Optional, repo to use for the network deployment +# Default: bigchaindb/bigchaindb +$ export STACK_REPO=bigchaindb/bigchaindb + +#Optional, codebase to use for the network deployment +# Default: master +$ export STACK_BRANCH=master + +#Optional, since 0.22.8 is the default tendermint version. +$ export TM_VERSION=0.22.8 + +#Optional, since 3.6 is the default MongoDB version. +$ export MONGO_VERSION=3.6 + +$ bash stack.sh +``` + +**Note**: For MacOSX users, the script will not install `Docker for Mac`, it only detects if `docker` is installed on the system, if not make sure to install [Docker for Mac](https://docs.docker.com/docker-for-mac/). Also make sure Docker API Version > 1.25. To check Docker API Version: + +`docker version --format '{{.Server.APIVersion}}'` + +### STACK_TYPE: local +This configuration deploys a VM based Planetmint network on your host/dev. All the services are running as processes on the VMs. For `local` deployments the following dependencies must be installed i.e. + +- Vagrant + - Vagrant plugins. + - vagrant-cachier + - vagrant-vbguest + - vagrant-hosts + - vagrant-azure + - `vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts vagrant-azure` +- [Virtualbox](https://www.virtualbox.org/wiki/Downloads) + +#### Example: VM +Deploy a 4 node VM based Planetmint network. + +```text +$ export STACK_TYPE=local + +# Optional, since 4 is the default size. +$ export STACK_SIZE=4 + +# Optional, default is 2048 +$ export STACK_VM_MEMORY=2048 + +#Optional, default is 1 +$ export STACK_VM_CPUS=1 + +#Optional, default is ubuntu/xenial64. Supported/tested images: bento/centos-7, fedora/25-cloud-base +$ export STACK_BOX_NAME=ubuntu/xenial64 + +#Optional, repo to use for the network deployment +# Default: bigchaindb/bigchaindb +$ export STACK_REPO=bigchaindb/bigchaindb + +#Optional, codebase to use for the network deployment +# Default: master +$ export STACK_BRANCH=master + +#Optional, since 0.22.8 is the default tendermint version +$ export TM_VERSION=0.22.8 + +#Optional, since 3.6 is the default MongoDB version. +$ export MONGO_VERSION=3.6 + +$ bash stack.sh +``` + +### STACK_TYPE: cloud + +This configuration deploys a docker based Planetmint network on a cloud instance. Currently, only Azure is supported. +For `cloud` deployments the following dependencies must be installed i.e. + +- Vagrant + - Vagrant plugins. + - vagrant-cachier + - vagrant-vbguest + - vagrant-hosts + - vagrant-azure + - `vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts vagrant-azure` + +#### Example: stack +Deploy a 4 node docker based Planetmint network on an Azure instance. + +- [Create an Azure Active Directory(AAD) Application](https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application) + +- Generate or specify an SSH keypair to login to the Azure instance. + + - **Example:** + ```text + $ ssh-keygen -t rsa -C "" -f /path/to/key/ + ``` + +- Configure parameters for `stack.sh` +```text + +# After creating the AAD application with access to Azure Resource +# Group Manager for your subscription, it will return a JSON object + +$ export AZURE_CLIENT_ID= + +$ export AZURE_TENANT_ID= + +# Can be retrieved via +# az account list --query "[?isDefault].id" -o tsv +$ export AZURE_SUBSCRIPTION_ID= + +$ export AZURE_CLIENT_SECRET= + +$ export STACK_TYPE=cloud + +# Currently on azure is supported +$ export STACK_TYPE_PROVIDER=azure + +$ export SSH_PRIVATE_KEY_PATH= + +# Optional, Azure region of the instance. Default: westeurope +$ export AZURE_REGION=westeurope + +# Optional, Azure image urn of the instance. Default: Canonical:UbuntuServer:16.04-LTS:latest +$ export AZURE_IMAGE_URN=Canonical:UbuntuServer:16.04-LTS:latest + +# Optional, Azure resource group. Default: bdb-vagrant-rg-yyyy-mm-dd(current date) +$ export AZURE_RESOURCE_GROUP=bdb-vagrant-rg-2018-01-01 + +# Optional, DNS prefix of the Azure instance. Default: bdb-instance-yyyy-mm-dd(current date) +$ export AZURE_DNS_PREFIX=bdb-instance-2018-01-01 + +# Optional, Admin username of the Azure instance. Default: vagrant +$ export AZURE_ADMIN_USERNAME=vagrant + +# Optional, Azure instance size. Default: Standard_D2_v2 +$ export AZURE_VM_SIZE=Standard_D2_v2 + +$ bash stack.sh +``` + +## Delete/Unstack a Planetmint network + +Export all the variables exported for the corresponding `stack.sh` script and +run `unstack.sh` to delete/remove/unstack the Planetmint network/stack. + +```text +$ bash unstack.sh + +OR + +# -s implies soft unstack. i.e. Only applicable for local and cloud based +# networks. Only deletes/stops the docker(s)/process(es) and does not +# delete the instances created via Vagrant or on Cloud. Default: hard +$ bash unstack.sh -s +``` diff --git a/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-node-as-processes.md b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-node-as-processes.md new file mode 100644 index 0000000..82b8bbb --- /dev/null +++ b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-node-as-processes.md @@ -0,0 +1,138 @@ + + +# Notes on Running a Local Dev Node as Processes + +The following doc describes how to run a local node for developing Planetmint Tendermint version. + +There are two crucial dependencies required to start a local node: + +- MongoDB +- Tendermint + +and of course you also need to install Planetmint Sever from the local code you just developed. + +## Install and Run MongoDB + +MongoDB can be easily installed, just refer to their [installation documentation](https://docs.mongodb.com/manual/installation/) for your distro. +We know MongoDB 3.4 and 3.6 work with Planetmint. +After the installation of MongoDB is complete, run MongoDB using `sudo mongod` + +## Install and Run Tendermint + +### Installing a Tendermint Executable + +The version of Planetmint Server described in these docs only works well with Tendermint 0.31.5 (not a higher version number). Install that: + +```bash +$ sudo apt install -y unzip +$ wget https://github.com/tendermint/tendermint/releases/download/v0.31.5/tendermint_v0.31.5_linux_amd64.zip +$ unzip tendermint_v0.31.5_linux_amd64.zip +$ rm tendermint_v0.31.5_linux_amd64.zip +$ sudo mv tendermint /usr/local/bin +``` + +### Installing Tendermint Using Docker + +Tendermint can be run directly using the docker image. Refer [here](https://hub.docker.com/r/tendermint/tendermint/) for more details. + +### Installing Tendermint from Source + +Before we can begin installing Tendermint one should ensure that the Golang is installed on system and `$GOPATH` should be set in the `.bashrc` or `.zshrc`. An example setup is shown below + +```bash + +$ echo $GOPATH +/home/user/Documents/go +$ go -h +Go is a tool for managing Go source code. + +Usage: + + go command [arguments] + +The commands are: + + build compile packages and dependencies + clean remove object files + +... + +``` + +- We can drop `GOPATH` in `PATH` so that installed Golang packages are directly available in the shell. To do that add the following to your `.bashrc` + +```bash +export PATH=${PATH}:${GOPATH}/bin +``` + +Follow [the Tendermint docs](https://tendermint.com/docs/introduction/install.html#from-source) to install Tendermint from source. + +If the installation is successful then Tendermint is installed at `$GOPATH/bin`. To ensure Tendermint's installed fine execute the following command, + +```bash +$ tendermint -h +Tendermint Core (BFT Consensus) in Go + +Usage: + tendermint [command] + +Available Commands: + gen_validator Generate new validator keypair + help Help about any command + init Initialize Tendermint +... + +``` + +### Running Tendermint + +- We can initialize and run tendermint as follows, +```bash +$ tendermint init +... + +$ tendermint node --consensus.create_empty_blocks=false +``` +The argument `--consensus.create_empty_blocks=false` specifies that Tendermint should not commit empty blocks. + + +- To reset all the data stored in Tendermint execute the following command, + +```bash +$ tendermint unsafe_reset_all +``` + +## Install Planetmint + +To install Planetmint from source (for dev), clone the repo and execute the following command, (it is better that you create a virtual env for this) + +```bash +$ git clone https://github.com/bigchaindb/bigchaindb.git +$ cd bigchaindb +$ pip install -e .[dev] # or pip install -e '.[dev]' # for zsh +``` + +## Running All Tests + +To execute tests when developing a feature or fixing a bug one could use the following command, + +```bash +$ pytest -v +``` + +NOTE: MongoDB and Tendermint should be running as discussed above. + +One could mark a specific test and execute the same by appending `-m my_mark` to the above command. + +Although the above should prove sufficient in most cases but in case tests are failing on Travis CI then the following command can be used to possibly replicate the failure locally, + +```bash +$ docker-compose run --rm --no-deps bdb pytest -v --cov=bigchaindb +``` + +NOTE: before executing the above command the user must ensure that they reset the Tendermint container by executing `tendermint usafe_reset_all` command in the Tendermint container. diff --git a/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-node-with-docker-compose.md b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-node-with-docker-compose.md new file mode 100644 index 0000000..4f85ad2 --- /dev/null +++ b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/run-node-with-docker-compose.md @@ -0,0 +1,108 @@ + + +# Notes on Running a Local Dev Node with Docker Compose + +## Setting up a single node development environment with ``docker-compose`` + +### Using the Planetmint 2.0 developer toolbox +We grouped all useful commands under a simple `Makefile`. + +Run a Planetmint node in the foreground: +```bash +$ make run +``` + +There are also other commands you can execute: +- `make start`: Run Planetmint from source and daemonize it (stop it with `make stop`). +- `make stop`: Stop Planetmint. +- `make logs`: Attach to the logs. +- `make test`: Run all unit and acceptance tests. +- `make test-unit-watch`: Run all tests and wait. Every time you change code, tests will be run again. +- `make cov`: Check code coverage and open the result in the browser. +- `make doc`: Generate HTML documentation and open it in the browser. +- `make clean`: Remove all build, test, coverage and Python artifacts. +- `make reset`: Stop and REMOVE all containers. WARNING: you will LOSE all data stored in Planetmint. + + +### Using `docker-compose` directly +The Planetmint `Makefile` is a wrapper around some `docker-compose` commands we use frequently. If you need a finer granularity to manage the containers, you can still use `docker-compose` directly. This part of the documentation explains how to do that. + +```bash +$ docker-compose build bigchaindb +$ docker-compose up -d bdb +``` + +The above command will launch all 3 main required services/processes: + +* ``mongodb`` +* ``tendermint`` +* ``bigchaindb`` + +To follow the logs of the ``tendermint`` service: + +```bash +$ docker-compose logs -f tendermint +``` + +To follow the logs of the ``bigchaindb`` service: + +```bash +$ docker-compose logs -f bigchaindb +``` + +To follow the logs of the ``mongodb`` service: + +```bash +$ docker-compose logs -f mdb +``` + +Simple health check: + +```bash +$ docker-compose up curl-client +``` + +Post and retrieve a transaction -- copy/paste a driver basic example of a +``CREATE`` transaction: + +```bash +$ docker-compose -f docker-compose.yml run --rm bdb-driver ipython +``` + +**TODO**: A python script to post and retrieve a transaction(s). + +### Running Tests + +Run all the tests using: + +```bash +$ docker-compose run --rm --no-deps bigchaindb pytest -v +``` + +Run tests from a file: + +```bash +$ docker-compose run --rm --no-deps bigchaindb pytest /path/to/file -v +``` + +Run specific tests: +```bash +$ docker-compose run --rm --no-deps bigchaindb pytest /path/to/file -k "" -v +``` + +### Building Docs + +You can also develop and build the Planetmint docs using ``docker-compose``: + +```bash +$ docker-compose build bdocs +$ docker-compose up -d bdocs +``` + +The docs will be hosted on port **33333**, and can be accessed over [localhost](http:/localhost:33333), [127.0.0.1](http:/127.0.0.1:33333) +OR http:/HOST_IP:33333. diff --git a/docs/root/source/contributing/dev-setup-coding-and-contribution-process/write-code.rst b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/write-code.rst new file mode 100644 index 0000000..ada8bbf --- /dev/null +++ b/docs/root/source/contributing/dev-setup-coding-and-contribution-process/write-code.rst @@ -0,0 +1,157 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Write Code +========== + +Know What You Want to Write Code to Do +-------------------------------------- + +Do you want to write code to resolve an open issue (bug)? Which one? + +Do you want to implement a Planetmint Enhancement Proposal (PEP)? Which one? + +You should know why you want to write code before you go any farther. + + +Refresh Yourself about the C4 Process +------------------------------------- + +C4 is the Collective Code Construction Contract. It's quite short: +`re-reading it will only take a few minutes `_. + + +Set Up Your Local Machine. Here's How. +-------------------------------------- + +- Make sure you have Git installed. + +- Get a text editor. Internally, we like: + + - Vim + - PyCharm + - Visual Studio Code + - Atom + - GNU Emacs (Trent is crazy) + - GNU nano (Troy has lost his mind) + +- If you plan to do JavaScript coding, get the latest JavaScript stuff (npm etc.). + +- If you plan to do Python coding, get `the latest Python `_, and + get the latest ``pip``. + +.. warning:: + + Don't use apt or apt-get to get the latest ``pip``. It won't work properly. Use ``get-pip.py`` + from `the pip website `_. + +- Use the latest ``pip`` to get the latest ``virtualenv``: + + .. code:: + + $ pip install virtualenv + +- Create a Python Virttual Environment (virtualenv) for doing Planetmint Server development. There are many ways to do that. Google around and pick one. + An old-fashioned but fine way is: + + .. code:: + + $ virtualenv -p $(which python3.6) NEW_ENV_NAME + $ . NEW_ENV_NAME/bin/activate + + Be sure to use Python 3.6.x as the Python version for your virtualenv. The virtualenv creation process will actually get the + latest ``pip``, ``wheel`` and ``setuptools`` and put them inside the new virtualenv. + + +Before You Start Writing Code +----------------------------- + +Read `BEP-24 `_ +so you know what to do to ensure that your changes (i.e. your future pull request) can be merged. +It's easy and will save you some hassle later on. + + +Start Writing Code +------------------ + +Use the Git `Fork and Pull Request Workflow `_. Tip: You could print that page for reference. + +Your Python code should follow `our Python Style Guide `_. +Similarly for JavaScript. + +Make sure `pre-commit `_ actually checks commits. Do: + + .. code:: + + $ pip install pre-commit # might not do anything if it is already installed, which is okay + $ pre-commit install + +That will load the pre-commit settings in the file ``.pre-commit-config.yaml``. Now every time you do ``git commit ``, pre-commit +will run all those checks. + +To install Planetmint Server from the local code, and to keep it up to date with the latest code in there, use: + + .. code:: + + $ pip install -e .[dev] + +The ``-e`` tells it to use the latest code. The ``.`` means use the current directory, which should be the one containing ``setup.py``. +The ``[dev]`` tells it to install some extra Python packages. Which ones? Open ``setup.py`` and look for ``dev`` in the ``extras_require`` section. + + +Remember to Write Tests +----------------------- + +We like to test everything, if possible. Unit tests and also integration tests. We use the `pytest `_ +framework to write Python tests. Read all about it. + +Most tests are in the ``tests/`` folder. Take a look around. + + +Running a Local Node/Network for Dev and Test +--------------------------------------------- + +This is tricky and personal. Different people do it different ways. We documented some of those on separate pages: + +- `Dev node setup and running all tests with Docker Compose `_ +- `Dev node setup and running all tests as processes `_ +- `Dev network setup with stack.sh `_ +- `Dev network setup with Ansible `_ +- More to come? + + +Create the PR on GitHub +----------------------- + +Git push your branch to GitHub so as to create a pull request against the branch where the code you want to change *lives*. + +Travis and Codecov will run and might complain. Look into the complaints and fix them before merging. +Travis gets its configuration and setup from the files: + +- Some environment variables, if they are used. See https://docs.travis-ci.com/user/environment-variables/ +- ``.travis.yml`` +- ``tox.ini`` - What is tox? See `tox.readthedocs.io `_ +- ``.ci/`` (as in Travis CI = Continuous Integration) + + - ``travis-after-success.sh`` + - ``travis-before-install.sh`` + - ``travis-before-script.sh`` + - ``travis-install.sh`` + - ``travis_script.sh`` + +Read about the `Travis CI build lifecycle `_ to understand when those scripts run and what they do. +You can have even more scripts! + +Codecov gets its configuration from the file `codeocov.yaml `_ which is also documented at +`docs.codecov.io `_. Codecov might also use ``setup.cfg``. + + +Merge! +------ + +Ideally, we like your PR and merge it right away. We don't want to keep you waiting. + +If we want to make changes, we'll do them in a follow-up PR. diff --git a/docs/root/source/contributing/index.rst b/docs/root/source/contributing/index.rst new file mode 100644 index 0000000..40038ee --- /dev/null +++ b/docs/root/source/contributing/index.rst @@ -0,0 +1,29 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Contributing to Planetmint +========================== + +There are many ways you can contribute to Planetmint. +It includes several sub-projects. + +- `Planetmint Server `_ +- `Planetmint Python Driver `_ +- `Planetmint JavaScript Driver `_ +- `Planetmint Java Driver `_ +- `cryptoconditions `_ (a Python package by us) +- `py-abci `_ (a Python package we use) +- `Planetmint Enhancement Proposals (PEPs) `_ + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + + ways-to-contribute/index + dev-setup-coding-and-contribution-process/index + cross-project-policies/index diff --git a/docs/root/source/contributing/ways-to-contribute/index.rst b/docs/root/source/contributing/ways-to-contribute/index.rst new file mode 100644 index 0000000..4f676eb --- /dev/null +++ b/docs/root/source/contributing/ways-to-contribute/index.rst @@ -0,0 +1,14 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Ways to Contribute +================== + +.. toctree:: + :maxdepth: 1 + + report-a-bug + write-docs diff --git a/docs/root/source/contributing/ways-to-contribute/report-a-bug.md b/docs/root/source/contributing/ways-to-contribute/report-a-bug.md new file mode 100644 index 0000000..7f5fbe9 --- /dev/null +++ b/docs/root/source/contributing/ways-to-contribute/report-a-bug.md @@ -0,0 +1,44 @@ + + +How to contribute? +================== + +# Report a Bug + +To report a bug, go to the relevant GitHub repository, click on the **Issues** tab, click on the **New issue** button, and read the instructions. + +# Write an Issue + +To write an issue, go to the relevant GitHub repository, click on the **Issues** tab, click on the **New issue** button, and read the instructions. + +# Answer Questions + +People ask questions about Planetmint in the following places: + +- Gitter + - [bigchaindb/bigchaindb](https://gitter.im/planetmint/community) + - [bigchaindb/js-bigchaindb-driver](https://gitter.im/planetmint/community)) +- [Twitter](https://twitter.com/planetmint) +- [Stack Overflow "bigchaindb"](https://stackoverflow.com/search?q=planetmint) + +Feel free to hang out and answer some questions. People will be thankful. + +# Write a Planetmint Enhancement Proposal (PEP) + +If you have an idea for a new feature or enhancement, and you want some feedback before you write a full Planetmint Enhancement Proposal (PEP), then feel free to: + - ask in the [planetmint/community Gitter chat room](https://gitter.im/planetmint/planetmint) or + - [open a new issue in the planetmint/PEPs repo](https://github.com/planetmint/PEPs/issues/new) and give it the label **PEP idea**. + +If you want to discuss an existing PEP, then [open a new issue in the planetmint/PEPs repo](https://github.com/planetmint/BEPs/issues/new) and give it the label **discuss existing PEP**. + +## Steps to Write a New PEP + +1. Look at the structure of existing PEPs in the [planetmint/PEPs repo](https://github.com/planetmint/PEPs). Note the section headings. [PEP-2](https://github.com/planetmint/PEPs/tree/master/2) (our variant of the consensus-oriented specification system [COSS]) says more about the expected structure and process. +1. Write a first draft of your PEP. It doesn't have to be long or perfect. +1. Push your BEP draft to the [planetmint/PEPs repo](https://github.com/planetmint/PEPs) and make a pull request. [PEP-1](https://github.com/planetmint/PEPs/tree/master/1) (our variant of C4) outlines the process we use to handle all pull requests. In particular, we try to merge all pull requests quickly. +1. Your PEP can be revised by pushing more pull requests. \ No newline at end of file diff --git a/docs/root/source/contributing/ways-to-contribute/write-docs.md b/docs/root/source/contributing/ways-to-contribute/write-docs.md new file mode 100644 index 0000000..cb6f018 --- /dev/null +++ b/docs/root/source/contributing/ways-to-contribute/write-docs.md @@ -0,0 +1,30 @@ + + +# Write Docs + +If you're writing code, you should also update any related docs. However, you might want to write docs only, such as: + +- General explainers +- Tutorials +- Courses +- Code explanations +- How Planetmint relates to other blockchain things +- News from recent events + +You can certainly do that! + +- The docs for Planetmint Server live under ``planetmint/docs/`` in the ``planetmint/planetmint`` repo. +- There are docs for the Python driver under ``planetmint-driver/docs/`` in the ``planetmint/planetmint-driver`` repo. +- There are docs for the JavaScript driver under ``planetmint/js-planetmint-driver`` in the ``planetmint/js-planetmint-driver`` repo. +- The source code for the Planetmint website is in a private repo, but we can give you access if you ask. + +The [Planetmint Transactions Specs](https://github.com/planetmint/PEPs/tree/master/tx-specs/) (one for each spec version) are in the ``planetmint/PEPs`` repo. + +You can write the docs using Markdown (MD) or RestructuredText (RST). Sphinx can understand both. RST is more powerful. + +ReadTheDocs will automatically rebuild the docs whenever a commit happens on the ``master`` branch, or on one of the other branches that it is monitoring. diff --git a/docs/root/source/drivers/index.rst b/docs/root/source/drivers/index.rst new file mode 100644 index 0000000..de99da7 --- /dev/null +++ b/docs/root/source/drivers/index.rst @@ -0,0 +1,31 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Drivers +======= + +Connectors to Planetmint are referred to as drivers within the community. A driver is used to create valid transactions, to generate key pairs, to sign transactions and to post the transaction to the Planetmint API. + +These drivers were originally created by the original Planetmint team: + +* `Python Driver `_ +* `JavaScript / Node.js Driver `_ +* `Java Driver `_ + +These drivers and tools were created by the Planetmint community: + +.. warning:: + + Some of these projects are a work in progress, + but may still be useful. + Others might not work with the latest version of Planetmint. + +* `ANSI C driver `_, should also work with C++ (working as of June 2019) +* `C# driver `_ (working as of May 2019) +* `Haskell transaction builder `_ +* `Go driver `_ +* `Ruby driver `_ +* `Ruby library for preparing/signing transactions and submitting them or querying a Planetmint node (MIT licensed) `_ diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst new file mode 100644 index 0000000..36bb193 --- /dev/null +++ b/docs/root/source/index.rst @@ -0,0 +1,33 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Planetmint Documentation +======================== + +Meet Planetmint. The blockchain database. + +It has some database characteristics and some blockchain `properties `_, +including decentralization, immutability and native support for assets. + +At a high level, one can communicate with a Planetmint network (set of nodes) using the Planetmint HTTP API, or a wrapper for that API, such as the Planetmint Python Driver. Each Planetmint node runs Planetmint Server and various other software. The `terminology page `_ explains some of those terms in more detail. + +More About Planetmint +--------------------- + +.. toctree:: + :maxdepth: 1 + + Planetmint Docs Home + about-bigchaindb + terminology + properties + basic-usage + installation/index + drivers/index + query + contributing/index + korean/index + diff --git a/docs/root/source/installation/_static/Conditions_Circuit_Diagram.png b/docs/root/source/installation/_static/Conditions_Circuit_Diagram.png new file mode 100644 index 0000000..2c29eda Binary files /dev/null and b/docs/root/source/installation/_static/Conditions_Circuit_Diagram.png differ diff --git a/docs/root/source/installation/_static/Node-components.png b/docs/root/source/installation/_static/Node-components.png new file mode 100644 index 0000000..4bc8e9a Binary files /dev/null and b/docs/root/source/installation/_static/Node-components.png differ diff --git a/docs/root/source/installation/_static/arch.jpg b/docs/root/source/installation/_static/arch.jpg new file mode 100644 index 0000000..570d63c Binary files /dev/null and b/docs/root/source/installation/_static/arch.jpg differ diff --git a/docs/root/source/installation/_static/cc_escrow_execute_abort.png b/docs/root/source/installation/_static/cc_escrow_execute_abort.png new file mode 100644 index 0000000..959f276 Binary files /dev/null and b/docs/root/source/installation/_static/cc_escrow_execute_abort.png differ diff --git a/docs/root/source/installation/_static/models_diagrams.odg b/docs/root/source/installation/_static/models_diagrams.odg new file mode 100644 index 0000000..058fa45 Binary files /dev/null and b/docs/root/source/installation/_static/models_diagrams.odg differ diff --git a/docs/root/source/installation/_static/mongodb_cloud_manager_1.png b/docs/root/source/installation/_static/mongodb_cloud_manager_1.png new file mode 100644 index 0000000..16073d6 Binary files /dev/null and b/docs/root/source/installation/_static/mongodb_cloud_manager_1.png differ diff --git a/docs/root/source/installation/_static/monitoring_system_diagram.png b/docs/root/source/installation/_static/monitoring_system_diagram.png new file mode 100644 index 0000000..6b654ea Binary files /dev/null and b/docs/root/source/installation/_static/monitoring_system_diagram.png differ diff --git a/docs/root/source/installation/_static/stories_3_assets.png b/docs/root/source/installation/_static/stories_3_assets.png new file mode 100644 index 0000000..fda19e8 Binary files /dev/null and b/docs/root/source/installation/_static/stories_3_assets.png differ diff --git a/docs/root/source/installation/_static/tx_escrow_execute_abort.png b/docs/root/source/installation/_static/tx_escrow_execute_abort.png new file mode 100644 index 0000000..3283d45 Binary files /dev/null and b/docs/root/source/installation/_static/tx_escrow_execute_abort.png differ diff --git a/docs/root/source/installation/_static/tx_multi_condition_multi_fulfillment_v1.png b/docs/root/source/installation/_static/tx_multi_condition_multi_fulfillment_v1.png new file mode 100644 index 0000000..1a38dd6 Binary files /dev/null and b/docs/root/source/installation/_static/tx_multi_condition_multi_fulfillment_v1.png differ diff --git a/docs/root/source/installation/_static/tx_schematics.odg b/docs/root/source/installation/_static/tx_schematics.odg new file mode 100644 index 0000000..49453f8 Binary files /dev/null and b/docs/root/source/installation/_static/tx_schematics.odg differ diff --git a/docs/root/source/installation/_static/tx_single_condition_single_fulfillment_v1.png b/docs/root/source/installation/_static/tx_single_condition_single_fulfillment_v1.png new file mode 100644 index 0000000..0479119 Binary files /dev/null and b/docs/root/source/installation/_static/tx_single_condition_single_fulfillment_v1.png differ diff --git a/docs/root/source/installation/api/http-client-server-api.rst b/docs/root/source/installation/api/http-client-server-api.rst new file mode 100644 index 0000000..c80d515 --- /dev/null +++ b/docs/root/source/installation/api/http-client-server-api.rst @@ -0,0 +1,737 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _the-http-client-server-api: + +The HTTP Client-Server API +========================== + +This page assumes you already know an API Root URL +for a Planetmint node or reverse proxy. +It should be something like ``https://example.com:9984`` +or ``https://12.34.56.78:9984``. + +If you set up a Planetmint node or reverse proxy yourself, +and you're not sure what the API Root URL is, +then see the last section of this page for help. + +.. _bigchaindb-root-url: + +Planetmint Root URL +------------------- + +If you send an HTTP GET request to the Planetmint Root URL +e.g. ``http://localhost:9984`` +or ``https://example.com:9984`` +(with no ``/api/v1/`` on the end), +then you should get an HTTP response +with something like the following in the body: + +.. literalinclude:: http-samples/index-response.http + :language: http + + +.. _api-root-endpoint: + +API Root Endpoint +----------------- + +If you send an HTTP GET request to the API Root Endpoint +e.g. ``http://localhost:9984/api/v1/`` +or ``https://example.com:9984/api/v1/``, +then you should get an HTTP response +that allows you to discover the Planetmint API endpoints: + +.. literalinclude:: http-samples/api-index-response.http + :language: http + + +Transactions Endpoint +--------------------- + +.. note:: + + If you want to do more sophisticated queries + than those provided by the Planetmint HTTP API, + then one option is to connect to MongoDB directly (if possible) + and do whatever queries MongoDB allows. + For more about that option, see + `the page about querying Planetmint `_. + +.. http:get:: /api/v1/transactions/{transaction_id} + + Get the transaction with the ID ``transaction_id``. + + If a transaction with ID ``transaction_id`` has been included + in a committed block, then this endpoint returns that transaction, + otherwise the response will be ``404 Not Found``. + + :param transaction_id: transaction ID + :type transaction_id: hex string + + **Example request**: + + .. literalinclude:: http-samples/get-tx-id-request.http + :language: http + + **Example response**: + + .. literalinclude:: http-samples/get-tx-id-response.http + :language: http + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A transaction with that ID was found. + :statuscode 404: A transaction with that ID was not found. + +.. http:get:: /api/v1/transactions + + Requests to the ``/api/v1/transactions`` endpoint + without any query parameters will get a response status code ``400 Bad Request``. + +.. http:get:: /api/v1/transactions?asset_id={asset_id}&operation={CREATE|TRANSFER}&last_tx={true|false} + + Get a list of transactions that use an asset with the ID ``asset_id``. + + If ``operation`` is ``CREATE``, then the CREATE transaction which created + the asset with ID ``asset_id`` will be returned. + + If ``operation`` is ``TRANSFER``, then every TRANSFER transaction involving + the asset with ID ``asset_id`` will be returned. + This allows users to query the entire history or + provenance of an asset. + + If ``operation`` is not included, then *every* transaction involving + the asset with ID ``asset_id`` will be returned. + + if ``last_tx`` is set to ``true``, only the last transaction is returned + instead of all transactions with the given ``asset_id``. + + This endpoint returns transactions only if they are in committed blocks. + + :query string operation: (Optional) ``CREATE`` or ``TRANSFER``. + + :query string asset_id: asset ID. + + :query string last_tx: (Optional) ``true`` or ``false``. + + + **Example request**: + + .. literalinclude:: http-samples/get-tx-by-asset-request.http + :language: http + + **Example response**: + + .. literalinclude:: http-samples/get-tx-by-asset-response.http + :language: http + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A list of transactions containing an asset with ID ``asset_id`` was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. + + +.. http:post:: /api/v1/transactions?mode={mode} + + This endpoint is used to send a transaction to a Planetmint network. + The transaction is put in the body of the request. + + :query string mode: (Optional) One of the three supported modes to send a transaction: ``async``, ``sync``, ``commit``. The default is ``async``. + + Once the posted transaction arrives at a Planetmint node, + that node will check to see if the transaction is valid. + If it's invalid, the node will return an HTTP 400 (error). + Otherwise, the node will send the transaction to Tendermint (in the same node) using the + `Tendermint broadcast API + `_. + + The meaning of the ``mode`` query parameter is inherited from the mode parameter in + `Tendermint's broadcast API + `_. + ``mode=async`` means the HTTP response will come back immediately, + before Tendermint asks Planetmint Server to check the validity of the transaction (a second time). + ``mode=sync`` means the HTTP response will come back + after Tendermint gets a response from Planetmint Server + regarding the validity of the transaction. + ``mode=commit`` means the HTTP response will come back once the transaction + is in a committed block. + + .. note:: + In the async and sync modes, after a successful HTTP response is returned, the transaction may still be rejected later on. All the transactions are recorded internally by Tendermint in WAL (Write-Ahead Log) before the HTTP response is returned. Nevertheless, the following should be noted: + + - Transactions in WAL including the failed ones are not exposed in any of the Planetmint or Tendermint APIs. + - Transactions are never fetched from WAL. WAL is never replayed. + - A critical failure (e.g. the system is out of disk space) may occur preventing transactions from being stored in WAL, even when the HTTP response indicates a success. + - If a transaction fails the validation because it conflicts with the other transactions of the same block, Tendermint includes it into its block, but Planetmint does not store these transactions and does not offer any information about them in the APIs. + + .. note:: + + The posted transaction should be valid. + The relevant + `Planetmint Transactions Spec `_ + explains how to build a valid transaction + and how to check if a transaction is valid. + One would normally use a driver such as the `Planetmint Python Driver + `_ + to build a valid transaction. + + .. note:: + + A client can subscribe to the + WebSocket Event Stream API + to listen for committed transactions. + + **Example request**: + + .. literalinclude:: http-samples/post-tx-request.http + :language: http + + **Example response**: + + .. literalinclude:: http-samples/post-tx-response.http + :language: http + + :resheader Content-Type: ``application/json`` + + :statuscode 202: The meaning of this response depends on the value + of the ``mode`` parameter. See above. + + :statuscode 400: The posted transaction was invalid. + + +.. http:post:: /api/v1/transactions + + This endpoint (without any parameters) will push a new transaction. + Since no ``mode`` parameter is included, the default mode is assumed: ``async``. + + +Transaction Outputs +------------------- + +The ``/api/v1/outputs`` endpoint returns transactions outputs filtered by a +given public key, and optionally filtered to only include either spent or +unspent outputs. + +.. note:: + + If you want to do more sophisticated queries + than those provided by the Planetmint HTTP API, + then one option is to connect to MongoDB directly (if possible) + and do whatever queries MongoDB allows. + For more about that option, see + `the page about querying Planetmint `_. + +.. http:get:: /api/v1/outputs + + Get transaction outputs by public key. The ``public_key`` parameter must be + a base58 encoded ed25519 public key associated with transaction output + ownership. + + Returns a list of transaction outputs. + + :param public_key: Base58 encoded public key associated with output + ownership. This parameter is mandatory and without it + the endpoint will return a ``400`` response code. + :param spent: (Optional) Boolean value (``true`` or ``false``) + indicating if the result set + should include only spent or only unspent outputs. If not + specified, the result includes all the outputs (both spent + and unspent) associated with the ``public_key``. + +.. http:get:: /api/v1/outputs?public_key={public_key} + + Return all outputs, both spent and unspent, for the ``public_key``. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/outputs?public_key=1AAAbbb...ccc HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "output_index": 0, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + }, + { + "output_index": 1, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } + ] + + :statuscode 200: A list of outputs was found and returned in the body of the response. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. + +.. http:get:: /api/v1/outputs?public_key={public_key}&spent=true + + Return all **spent** outputs for ``public_key``. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/outputs?public_key=1AAAbbb...ccc&spent=true HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "output_index": 0, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } + ] + + :statuscode 200: A list of outputs were found and returned in the body of the response. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. + +.. http:get:: /api/v1/outputs?public_key={public_key}&spent=false + + Return all **unspent** outputs for ``public_key``. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/outputs?public_key=1AAAbbb...ccc&spent=false HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "output_index": 1, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } + ] + + :statuscode 200: A list of outputs were found and returned in the body of the response. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. + + +Assets +------ + +.. note:: + + If you want to do more sophisticated queries + than those provided by the Planetmint HTTP API, + then one option is to connect to MongoDB directly (if possible) + and do whatever queries MongoDB allows. + For more about that option, see + `the page about querying Planetmint `_. + +.. http:get:: /api/v1/assets + + Return all the assets that match a given text search. + + :query string search: Text search string to query. + :query int limit: (Optional) Limit the number of returned assets. Defaults + to ``0`` meaning return all matching assets. + +.. http:get:: /api/v1/assets/?search={search} + + Return all assets that match a given text search. + + .. note:: + + The ``id`` of the asset + is the same ``id`` of the CREATE transaction that created the asset. + + .. note:: + + You can use ``assets/?search`` or ``assets?search``. + + If no assets match the text search it returns an empty list. + + If the text string is empty or the server does not support text search, + a ``400 Bad Request`` is returned. + + The results are sorted by text score. + For more information about the behavior of text search, see `MongoDB text + search behavior `_. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/assets/?search=bigchaindb HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "data": {"msg": "Hello Planetmint 1!"}, + "id": "51ce82a14ca274d43e4992bbce41f6fdeb755f846e48e710a3bbb3b0cf8e4204" + }, + { + "data": {"msg": "Hello Planetmint 2!"}, + "id": "b4e9005fa494d20e503d916fa87b74fe61c079afccd6e084260674159795ee31" + }, + { + "data": {"msg": "Hello Planetmint 3!"}, + "id": "fa6bcb6a8fdea3dc2a860fcdc0e0c63c9cf5b25da8b02a4db4fb6a2d36d27791" + } + ] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully. + :statuscode 400: The query was not executed successfully. Returned if the + text string is empty or the server does not support + text search. + +.. http:get:: /api/v1/assets?search={search}&limit={n_documents} + + Return at most ``n_documents`` assets that match a given text search. + + If no assets match the text search it returns an empty list. + + If the text string is empty or the server does not support text search, + a ``400 Bad Request`` is returned. + + The results are sorted by text score. + For more information about the behavior of text search, see `MongoDB text + search behavior `_. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/assets?search=bigchaindb&limit=2 HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "data": {"msg": "Hello Planetmint 1!"}, + "id": "51ce82a14ca274d43e4992bbce41f6fdeb755f846e48e710a3bbb3b0cf8e4204" + }, + { + "data": {"msg": "Hello Planetmint 2!"}, + "id": "b4e9005fa494d20e503d916fa87b74fe61c079afccd6e084260674159795ee31" + }, + ] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully. + :statuscode 400: The query was not executed successfully. Returned if the + text string is empty or the server does not support + text search. + + +Transaction Metadata +-------------------- + +.. note:: + + If you want to do more sophisticated queries + than those provided by the Planetmint HTTP API, + then one option is to connect to MongoDB directly (if possible) + and do whatever queries MongoDB allows. + For more about that option, see + `the page about querying Planetmint `_. + +.. http:get:: /api/v1/metadata + + Return all the metadata objects that match a given text search. + + :query string search: Text search string to query. + :query int limit: (Optional) Limit the number of returned metadata objects. Defaults + to ``0`` meaning return all matching objects. + +.. http:get:: /api/v1/metadata/?search={search} + + Return all metadata objects that match a given text search. + + .. note:: + + The ``id`` of the metadata + is the same ``id`` of the transaction where it was defined. + + .. note:: + + You can use ``metadata/?search`` or ``metadata?search``. + + If no metadata objects match the text search it returns an empty list. + + If the text string is empty or the server does not support text search, + a ``400 Bad Request`` is returned. + + The results are sorted by text score. + For more information about the behavior of text search, see `MongoDB text + search behavior `_. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/metadata/?search=bigchaindb HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "metadata": {"metakey1": "Hello Planetmint 1!"}, + "id": "51ce82a14ca274d43e4992bbce41f6fdeb755f846e48e710a3bbb3b0cf8e4204" + }, + { + "metadata": {"metakey2": "Hello Planetmint 2!"}, + "id": "b4e9005fa494d20e503d916fa87b74fe61c079afccd6e084260674159795ee31" + }, + { + "metadata": {"metakey3": "Hello Planetmint 3!"}, + "id": "fa6bcb6a8fdea3dc2a860fcdc0e0c63c9cf5b25da8b02a4db4fb6a2d36d27791" + } + ] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully. + :statuscode 400: The query was not executed successfully. Returned if the + text string is empty or the server does not support + text search. + +.. http:get:: /api/v1/metadata/?search={search}&limit={n_documents} + + Return at most ``n_documents`` metadata objects that match a given text search. + + If no metadata objects match the text search it returns an empty list. + + If the text string is empty or the server does not support text search, + a ``400 Bad Request`` is returned. + + The results are sorted by text score. + For more information about the behavior of text search, see `MongoDB text + search behavior `_. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/metadata?search=bigchaindb&limit=2 HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "metadata": {"msg": "Hello Planetmint 1!"}, + "id": "51ce82a14ca274d43e4992bbce41f6fdeb755f846e48e710a3bbb3b0cf8e4204" + }, + { + "metadata": {"msg": "Hello Planetmint 2!"}, + "id": "b4e9005fa494d20e503d916fa87b74fe61c079afccd6e084260674159795ee31" + }, + ] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully. + :statuscode 400: The query was not executed successfully. Returned if the + text string is empty or the server does not support + text search. + + +Validators +-------------------- + +.. http:get:: /api/v1/validators + + Return the local validators set of a given node. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/validators HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "pub_key": { + "data":"4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A", + "type":"ed25519" + }, + "power": 10 + }, + { + "pub_key": { + "data":"608D839D7100466D6BA6BE79C320F8B81DE93CFAA58CF9768CF921C6371F2553", + "type":"ed25519" + }, + "power": 5 + } + ] + + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully and validators set was returned. + + +Blocks +------ + +.. http:get:: /api/v1/blocks/{block_height} + + Get the block with the height ``block_height``. + + :param block_height: block height + :type block_height: integer + + **Example request**: + + .. literalinclude:: http-samples/get-block-request.http + :language: http + + **Example response**: + + .. literalinclude:: http-samples/get-block-response.http + :language: http + + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A block with that block height was found. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_height``. + :statuscode 404: A block with that block height was not found. + + +.. http:get:: /api/v1/blocks + + The unfiltered ``/blocks`` endpoint without any query parameters + returns a ``400 Bad Request`` status code. + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/blocks HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 400 Bad Request + + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_height``. + + +.. http:get:: /api/v1/blocks?transaction_id={transaction_id} + + Retrieve a list of block IDs (block heights), such that the blocks with those IDs contain a transaction with the ID ``transaction_id``. A correct response may consist of an empty list or a list with one block ID. + + .. note:: + In case no block was found, an empty list and an HTTP status code + ``200 OK`` is returned, as the request was still successful. + + :query string transaction_id: (Required) transaction ID + + **Example request**: + + .. literalinclude:: http-samples/get-block-txid-request.http + :language: http + + **Example response**: + + .. literalinclude:: http-samples/get-block-txid-response.http + :language: http + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The request was properly formed and zero or more blocks were found containing the specified ``transaction_id``. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``transaction_id``. + + +.. _determining-the-api-root-url: + +Determining the API Root URL +---------------------------- + +When you start Planetmint Server using ``bigchaindb start``, +an HTTP API is exposed at some address. The default is: + +``http://localhost:9984/api/v1/`` + +It's bound to ``localhost``, +so you can access it from the same machine, +but it won't be directly accessible from the outside world. +(The outside world could connect via a SOCKS proxy or whatnot.) + +The documentation about Planetmint Server :doc:`Configuration Settings <../../installation/node-setup/configuration>` +has a section about how to set ``server.bind`` so as to make +the HTTP API publicly accessible. + +If the API endpoint is publicly accessible, +then the public API Root URL is determined as follows: + +- The public IP address (like 12.34.56.78) + is the public IP address of the machine exposing + the HTTP API to the public internet (e.g. either the machine hosting + Gunicorn or the machine running the reverse proxy such as NGINX). + It's determined by AWS, Azure, Rackspace, or whoever is hosting the machine. + +- The DNS hostname (like example.com) is determined by DNS records, + such as an "A Record" associating example.com with 12.34.56.78 + +- The port (like 9984) is determined by the ``server.bind`` setting + if Gunicorn is exposed directly to the public Internet. + If a reverse proxy (like NGINX) is exposed directly to the public Internet + instead, then it could expose the HTTP API on whatever port it wants to. + (It should expose the HTTP API on port 9984, but it's not bound to do + that by anything other than convention.) diff --git a/docs/root/source/installation/api/http-samples/api-index-response.http b/docs/root/source/installation/api/http-samples/api-index-response.http new file mode 100644 index 0000000..f425f83 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/api-index-response.http @@ -0,0 +1,13 @@ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "assets": "/assets/", + "blocks": "/blocks/", + "docs": "https://docs.bigchaindb.com/projects/server/en/v2.2.2/http-client-server-api.html", + "metadata": "/metadata/", + "outputs": "/outputs/", + "streams": "ws://localhost:9985/api/v1/streams/valid_transactions", + "transactions": "/transactions/", + "validators": "/validators" +} diff --git a/docs/root/source/installation/api/http-samples/get-block-request.http b/docs/root/source/installation/api/http-samples/get-block-request.http new file mode 100644 index 0000000..596e19b --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-block-request.http @@ -0,0 +1,3 @@ +GET /api/v1/blocks/1 HTTP/1.1 +Host: example.com + diff --git a/docs/root/source/installation/api/http-samples/get-block-response.http b/docs/root/source/installation/api/http-samples/get-block-response.http new file mode 100644 index 0000000..b517f7d --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-block-response.http @@ -0,0 +1,45 @@ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "height": 1, + "transactions": [ + { + "asset": { + "data": { + "msg": "Hello Planetmint!" + } + }, + "id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317", + "inputs": [ + { + "fulfillment": "pGSAIDE5i63cn4X8T8N1sZ2mGkJD5lNRnBM4PZgI_zvzbr-cgUCy4BR6gKaYT-tdyAGPPpknIqI4JYQQ-p2nCg3_9BfOI-15vzldhyz-j_LZVpqAlRmbTzKS-Q5gs7ZIFaZCA_UD", + "fulfills": null, + "owners_before": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "metadata": { + "sequence": 0 + }, + "operation": "CREATE", + "outputs": [ + { + "amount": "1", + "condition": { + "details": { + "public_key": "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD", + "type": "ed25519-sha-256" + }, + "uri": "ni:///sha-256;PNYwdxaRaNw60N6LDFzOWO97b8tJeragczakL8PrAPc?fpt=ed25519-sha-256&cost=131072" + }, + "public_keys": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "version": "2.0" + } + ] +} diff --git a/docs/root/source/installation/api/http-samples/get-block-txid-request.http b/docs/root/source/installation/api/http-samples/get-block-txid-request.http new file mode 100644 index 0000000..1f17c6b --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-block-txid-request.http @@ -0,0 +1,3 @@ +GET /api/v1/blocks?transaction_id=4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317 HTTP/1.1 +Host: example.com + diff --git a/docs/root/source/installation/api/http-samples/get-block-txid-response.http b/docs/root/source/installation/api/http-samples/get-block-txid-response.http new file mode 100644 index 0000000..377f0e7 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-block-txid-response.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Content-Type: application/json + +[ + 1 +] diff --git a/docs/root/source/installation/api/http-samples/get-tx-by-asset-request.http b/docs/root/source/installation/api/http-samples/get-tx-by-asset-request.http new file mode 100644 index 0000000..a5c8f3f --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-tx-by-asset-request.http @@ -0,0 +1,3 @@ +GET /api/v1/transactions?operation=TRANSFER&asset_id=4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317 HTTP/1.1 +Host: example.com + diff --git a/docs/root/source/installation/api/http-samples/get-tx-by-asset-response.http b/docs/root/source/installation/api/http-samples/get-tx-by-asset-response.http new file mode 100644 index 0000000..63f9912 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-tx-by-asset-response.http @@ -0,0 +1,79 @@ +HTTP/1.1 200 OK +Content-Type: application/json + +[{ + "asset": { + "id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317" + }, + "id": "79ef6803210c941903d63d08b40fa17f0a5a04f11ac0ff04451553a187d97a30", + "inputs": [ + { + "fulfillment": "pGSAIDE5i63cn4X8T8N1sZ2mGkJD5lNRnBM4PZgI_zvzbr-cgUAYRI8kzKaZcrW-_avQrAIk5q-7o_7U6biBvoHk1ioBLqHSBcE_PAdNEaeWesAAW_HeCqNUWKaJ5Lzo5Nfz7QgN", + "fulfills": { + "output_index": 0, + "transaction_id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317" + }, + "owners_before": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "metadata": { + "sequence": 1 + }, + "operation": "TRANSFER", + "outputs": [ + { + "amount": "1", + "condition": { + "details": { + "public_key": "3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9", + "type": "ed25519-sha-256" + }, + "uri": "ni:///sha-256;lu6ov4AKkee6KWGnyjOVLBeyuP0bz4-O6_dPi15eYUc?fpt=ed25519-sha-256&cost=131072" + }, + "public_keys": [ + "3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9" + ] + } + ], + "version": "2.0" +}, +{ + "asset": { + "id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317" + }, + "id": "1fec726a3b426498147f1a1f19a92c187d551a7f66db4b88d666d7dcc10e86a4", + "inputs": [ + { + "fulfillment": "pGSAICw7Ul-c2lG6NFbHp3FbKRC7fivQcNGO7GS4wV3A-1QggUARCMty2JBK_OyPJntWEFxDG4-VbKMy853NtqwnPib5QUJIuwPQa1Y4aN2iIBuoqGE85Pmjcc1ScG9FCPSQHacK", + "fulfills": { + "output_index": 0, + "transaction_id": "79ef6803210c941903d63d08b40fa17f0a5a04f11ac0ff04451553a187d97a30" + }, + "owners_before": [ + "3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9" + ] + } + ], + "metadata": { + "sequence": 2 + }, + "operation": "TRANSFER", + "outputs": [ + { + "amount": "1", + "condition": { + "details": { + "public_key": "3Af3fhhjU6d9WecEM9Uw5hfom9kNEwE7YuDWdqAUssqm", + "type": "ed25519-sha-256" + }, + "uri": "ni:///sha-256;Ll1r0LzgHUvWB87yIrNFYo731MMUEypqvrbPATTbuD4?fpt=ed25519-sha-256&cost=131072" + }, + "public_keys": [ + "3Af3fhhjU6d9WecEM9Uw5hfom9kNEwE7YuDWdqAUssqm" + ] + } + ], + "version": "2.0" +}] diff --git a/docs/root/source/installation/api/http-samples/get-tx-id-request.http b/docs/root/source/installation/api/http-samples/get-tx-id-request.http new file mode 100644 index 0000000..87bd123 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-tx-id-request.http @@ -0,0 +1,3 @@ +GET /api/v1/transactions/4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317 HTTP/1.1 +Host: example.com + diff --git a/docs/root/source/installation/api/http-samples/get-tx-id-response.http b/docs/root/source/installation/api/http-samples/get-tx-id-response.http new file mode 100644 index 0000000..687aa33 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/get-tx-id-response.http @@ -0,0 +1,40 @@ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "asset": { + "data": { + "msg": "Hello Planetmint!" + } + }, + "id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317", + "inputs": [ + { + "fulfillment": "pGSAIDE5i63cn4X8T8N1sZ2mGkJD5lNRnBM4PZgI_zvzbr-cgUCy4BR6gKaYT-tdyAGPPpknIqI4JYQQ-p2nCg3_9BfOI-15vzldhyz-j_LZVpqAlRmbTzKS-Q5gs7ZIFaZCA_UD", + "fulfills": null, + "owners_before": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "metadata": { + "sequence": 0 + }, + "operation": "CREATE", + "outputs": [ + { + "amount": "1", + "condition": { + "details": { + "public_key": "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD", + "type": "ed25519-sha-256" + }, + "uri": "ni:///sha-256;PNYwdxaRaNw60N6LDFzOWO97b8tJeragczakL8PrAPc?fpt=ed25519-sha-256&cost=131072" + }, + "public_keys": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "version": "2.0" +} diff --git a/docs/root/source/installation/api/http-samples/index-response.http b/docs/root/source/installation/api/http-samples/index-response.http new file mode 100644 index 0000000..0825b18 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/index-response.http @@ -0,0 +1,20 @@ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "api": { + "v1": { + "assets": "/api/v1/assets/", + "blocks": "/api/v1/blocks/", + "docs": "https://docs.bigchaindb.com/projects/server/en/v2.2.2/http-client-server-api.html", + "metadata": "/api/v1/metadata/", + "outputs": "/api/v1/outputs/", + "streams": "ws://localhost:9985/api/v1/streams/valid_transactions", + "transactions": "/api/v1/transactions/", + "validators": "/api/v1/validators" + } + }, + "docs": "https://docs.bigchaindb.com/projects/server/en/v2.2.2/", + "software": "Planetmint", + "version": "2.2.2" +} diff --git a/docs/root/source/installation/api/http-samples/post-tx-request.http b/docs/root/source/installation/api/http-samples/post-tx-request.http new file mode 100644 index 0000000..53b0ae0 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/post-tx-request.http @@ -0,0 +1,41 @@ +POST /api/v1/transactions?mode=async HTTP/1.1 +Host: example.com +Content-Type: application/json + +{ + "asset": { + "data": { + "msg": "Hello Planetmint!" + } + }, + "id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317", + "inputs": [ + { + "fulfillment": "pGSAIDE5i63cn4X8T8N1sZ2mGkJD5lNRnBM4PZgI_zvzbr-cgUCy4BR6gKaYT-tdyAGPPpknIqI4JYQQ-p2nCg3_9BfOI-15vzldhyz-j_LZVpqAlRmbTzKS-Q5gs7ZIFaZCA_UD", + "fulfills": null, + "owners_before": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "metadata": { + "sequence": 0 + }, + "operation": "CREATE", + "outputs": [ + { + "amount": "1", + "condition": { + "details": { + "public_key": "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD", + "type": "ed25519-sha-256" + }, + "uri": "ni:///sha-256;PNYwdxaRaNw60N6LDFzOWO97b8tJeragczakL8PrAPc?fpt=ed25519-sha-256&cost=131072" + }, + "public_keys": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "version": "2.0" +} diff --git a/docs/root/source/installation/api/http-samples/post-tx-response.http b/docs/root/source/installation/api/http-samples/post-tx-response.http new file mode 100644 index 0000000..ed60767 --- /dev/null +++ b/docs/root/source/installation/api/http-samples/post-tx-response.http @@ -0,0 +1,40 @@ +HTTP/1.1 202 Accepted +Content-Type: application/json + +{ + "asset": { + "data": { + "msg": "Hello Planetmint!" + } + }, + "id": "4957744b3ac54434b8270f2c854cc1040228c82ea4e72d66d2887a4d3e30b317", + "inputs": [ + { + "fulfillment": "pGSAIDE5i63cn4X8T8N1sZ2mGkJD5lNRnBM4PZgI_zvzbr-cgUCy4BR6gKaYT-tdyAGPPpknIqI4JYQQ-p2nCg3_9BfOI-15vzldhyz-j_LZVpqAlRmbTzKS-Q5gs7ZIFaZCA_UD", + "fulfills": null, + "owners_before": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "metadata": { + "sequence": 0 + }, + "operation": "CREATE", + "outputs": [ + { + "amount": "1", + "condition": { + "details": { + "public_key": "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD", + "type": "ed25519-sha-256" + }, + "uri": "ni:///sha-256;PNYwdxaRaNw60N6LDFzOWO97b8tJeragczakL8PrAPc?fpt=ed25519-sha-256&cost=131072" + }, + "public_keys": [ + "4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD" + ] + } + ], + "version": "2.0" +} diff --git a/docs/root/source/installation/api/index.rst b/docs/root/source/installation/api/index.rst new file mode 100644 index 0000000..7693fab --- /dev/null +++ b/docs/root/source/installation/api/index.rst @@ -0,0 +1,16 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + + + +API +=== + +.. toctree:: + :maxdepth: 1 + + http-client-server-api + websocket-event-stream-api diff --git a/docs/root/source/installation/api/websocket-event-stream-api.rst b/docs/root/source/installation/api/websocket-event-stream-api.rst new file mode 100644 index 0000000..0978f4c --- /dev/null +++ b/docs/root/source/installation/api/websocket-event-stream-api.rst @@ -0,0 +1,110 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _the-websocket-event-stream-api: + +The WebSocket Event Stream API +============================== + +.. important:: + The WebSocket Event Stream runs on a different port than the Web API. The + default port for the Web API is `9984`, while the one for the Event Stream + is `9985`. + +Planetmint provides real-time event streams over the WebSocket protocol with +the Event Stream API. +Connecting to an event stream from your application enables a Planetmint node +to notify you as events occur, such as new `valid transactions <#valid-transactions>`_. + + +Demoing the API +--------------- + +You may be interested in demoing the Event Stream API with the `WebSocket echo test `_ +to familiarize yourself before attempting an integration. + + +Determining Support for the Event Stream API +-------------------------------------------- + +It's a good idea to make sure that the node you're connecting with +has advertised support for the Event Stream API. To do so, send a HTTP GET +request to the node's :ref:`api-root-endpoint` +(e.g. ``http://localhost:9984/api/v1/``) and check that the +response contains a ``streams`` property: + +.. code:: JSON + + { + ..., + "streams": "ws://example.com:9985/api/v1/streams/valid_transactions", + ... + } + + +Connection Keep-Alive +--------------------- + +The Event Stream API supports Ping/Pong frames as descibed in +`RFC 6455 `_. + +.. note:: + + It might not be possible to send PING/PONG frames via web browsers because + of non availability of Javascript API on different browsers to achieve the + same. + +Streams +------- + +Each stream is meant as a unidirectional communication channel, where the +Planetmint node is the only party sending messages. Any messages sent to the +Planetmint node will be ignored. + +Streams will always be under the WebSocket protocol (so ``ws://`` or +``wss://``) and accessible as extensions to the ``/api/v/streams/`` +API root URL (for example, valid transactions +would be accessible under ``/api/v1/streams/valid_transactions``). If you're +running your own Planetmint instance and need help determining its root URL, +then see the page titled :ref:`determining-the-api-root-url`. + +All messages sent in a stream are in the JSON format. + +.. note:: + + For simplicity, Planetmint initially only provides a stream for all + committed transactions. In the future, we may provide streams for other + information. We may + also provide the ability to filter the stream for specific qualities, such + as a specific ``output``'s ``public_key``. + + If you have specific use cases that you think would fit as part of this + API, consider creating a new `BEP `_. + +Valid Transactions +~~~~~~~~~~~~~~~~~~ + +``/valid_transactions`` + +Streams an event for any newly valid transactions committed to a block. Message +bodies contain the transaction's ID, associated asset ID, and containing +block's height. + +Example message: + +.. code:: JSON + + { + "transaction_id": "", + "asset_id": "", + "height": + } + + +.. note:: + + Transactions in Planetmint are committed in batches ("blocks") and will, + therefore, be streamed in batches. diff --git a/docs/root/source/installation/appendices/cryptography.rst b/docs/root/source/installation/appendices/cryptography.rst new file mode 100644 index 0000000..2416089 --- /dev/null +++ b/docs/root/source/installation/appendices/cryptography.rst @@ -0,0 +1,14 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Cryptography +============ + +Use the following link to find the Planetmint Transactions Spec (or Specs) that are relevant to you: + +`Planetmint Transactions Specs `_ + +Then see the sections titled **Cryptographic Hashes** and **Cryptographic Keys and Signatures**. \ No newline at end of file diff --git a/docs/root/source/installation/appendices/firewall-notes.md b/docs/root/source/installation/appendices/firewall-notes.md new file mode 100644 index 0000000..4b8ec39 --- /dev/null +++ b/docs/root/source/installation/appendices/firewall-notes.md @@ -0,0 +1,79 @@ + + +# Notes for Firewall Setup + +This is a page of notes on the ports potentially used by Planetmint nodes and the traffic they should expect, to help with firewall setup (or security group setup on cloud providers). This page is _not_ a firewall tutorial or step-by-step guide. + +## Expected Unsolicited Inbound Traffic + +The following ports should expect unsolicited inbound traffic: + +1. **Port 22** can expect inbound SSH (TCP) traffic from the node administrator (i.e. a small set of IP addresses). +1. **Port 9984** can expect inbound HTTP (TCP) traffic from Planetmint clients sending transactions to the Planetmint HTTP API. +1. **Port 9985** can expect inbound WebSocket traffic from Planetmint clients. +1. **Port 26656** can expect inbound Tendermint P2P traffic from other Tendermint peers. +1. **Port 9986** can expect inbound HTTP (TCP) traffic from clients accessing the Public Key of a Tendermint instance. + +All other ports should only get inbound traffic in response to specific requests from inside the node. + +## Port 22 + +Port 22 is the default SSH port (TCP) so you'll at least want to make it possible to SSH in from your remote machine(s). + +## Port 53 + +Port 53 is the default DNS port (UDP). It may be used, for example, by some package managers when look up the IP address associated with certain package sources. + +## Port 80 + +Port 80 is the default HTTP port (TCP). It's used by some package managers to get packages. It's _not_ the default port for the Planetmint client-server HTTP API. + +## Port 123 + +Port 123 is the default NTP port (UDP). You should be running an NTP daemon on production Planetmint nodes. NTP daemons must be able to send requests to external NTP servers and accept the respones. + +## Port 161 + +Port 161 is the default SNMP port (usually UDP, sometimes TCP). SNMP is used, for example, by some server monitoring systems. + +## Port 443 + +Port 443 is the default HTTPS port (TCP). Package managers might also get some packages using HTTPS. + +## Port 9984 + +Port 9984 is the default port for the Planetmint client-server HTTP API (TCP), which is served by Gunicorn HTTP Server. It's _possible_ allow port 9984 to accept inbound traffic from anyone, but we recommend against doing that. Instead, set up a reverse proxy server (e.g. using Nginx) and only allow traffic from there. Information about how to do that can be found [in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) + +If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [Planetmint Configuration Settings](../../installation/node-setup/configuration), where PORT is whatever port you chose (e.g. 9983). + +You may want to have Gunicorn and the reverse proxy running on different servers, so that both can listen on port 9984. That would also help isolate the effects of a denial-of-service attack. + +## Port 9985 + +Port 9985 is the default port for the Planetmint WebSocket Event Stream API. + +## Port 9986 + +Port 9986 is the default port to access the Public Key of a Tendermint instance, it is used by a NGINX instance +that runs with Tendermint instance(Pod), and only hosts the Public Key. + +## Port 26656 + +Port 26656 is the default port used by Tendermint Core to communicate with other instances of Tendermint Core (peers). + +## Port 26657 + +Port 26657 is the default port used by Tendermint Core for RPC traffic. Planetmint nodes use that internally; they don't expect incoming traffic from the outside world on port 26657. + +## Port 26658 + +Port 26658 is the default port used by Tendermint Core for ABCI traffic. Planetmint nodes use that internally; they don't expect incoming traffic from the outside world on port 26658. + +## Other Ports + +On Linux, you can use commands such as `netstat -tunlp` or `lsof -i` to get a sense of currently open/listening ports and connections, and the associated processes. diff --git a/docs/root/source/installation/appendices/generate-key-pair-for-ssh.md b/docs/root/source/installation/appendices/generate-key-pair-for-ssh.md new file mode 100644 index 0000000..95f49b9 --- /dev/null +++ b/docs/root/source/installation/appendices/generate-key-pair-for-ssh.md @@ -0,0 +1,41 @@ + + +# Generate a Key Pair for SSH + +This page describes how to use `ssh-keygen` +to generate a public/private RSA key pair +that can be used with SSH. +(Note: `ssh-keygen` is found on most Linux and Unix-like +operating systems; if you're using Windows, +then you'll have to use another tool, +such as PuTTYgen.) + +By convention, SSH key pairs get stored in the `~/.ssh/` directory. +Check what keys you already have there: +```text +ls -1 ~/.ssh/ +``` + +Next, make up a new key pair name (called `` below). +Here are some ideas: + +* `aws-bdb-2` +* `tim-bdb-azure` +* `chris-bcdb-key` + +Next, generate a public/private RSA key pair with that name: +```text +ssh-keygen -t rsa -C "" -f ~/.ssh/ +``` + +It will ask you for a passphrase. +You can use whatever passphrase you like, but don't lose it. +Two keys (files) will be created in `~/.ssh/`: + +1. `~/.ssh/.pub` is the public key +2. `~/.ssh/` is the private key diff --git a/docs/root/source/installation/appendices/index.rst b/docs/root/source/installation/appendices/index.rst new file mode 100755 index 0000000..97e9058 --- /dev/null +++ b/docs/root/source/installation/appendices/index.rst @@ -0,0 +1,19 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Appendices +========== + +.. toctree:: + :maxdepth: 1 + + generate-key-pair-for-ssh + cryptography + firewall-notes + ntp-notes + log-rotation + licenses + diff --git a/docs/root/source/installation/appendices/licenses.md b/docs/root/source/installation/appendices/licenses.md new file mode 100644 index 0000000..3a3b5e8 --- /dev/null +++ b/docs/root/source/installation/appendices/licenses.md @@ -0,0 +1,10 @@ + + +# Licenses + +Information about how the Planetmint Server code and documentation are licensed can be found in [the LICENSES.md file](https://github.com/bigchaindb/bigchaindb/blob/master/LICENSES.md) of the bigchaindb/bigchaindb repository on GitHub. diff --git a/docs/root/source/installation/appendices/log-rotation.md b/docs/root/source/installation/appendices/log-rotation.md new file mode 100644 index 0000000..0f453c2 --- /dev/null +++ b/docs/root/source/installation/appendices/log-rotation.md @@ -0,0 +1,50 @@ + + +# Logging and Log Rotation + +Each Planetmint node runs: + +- MongoDB +- Planetmint Server +- Tendermint + +When running a Planetmint node for long periods +of time, we need to consider doing log rotation, i.e. we do not want the logs taking +up large amounts of storage and making the node unresponsive or getting it into a bad state. + +## MongoDB Logging and Log Rotation + +See the MongoDB docs about +[logging](https://docs.mongodb.com/v3.6/administration/monitoring/#monitoring-standard-loggging) +and [log rotation](https://docs.mongodb.com/v3.6/tutorial/rotate-log-files/). + +## Planetmint Server Logging and Log Rotation + +Planetmint Server writes its logs to two files: normal logs and error logs. The names of those files, and their locations, are set as part of the Planetmint configuration settings. The default names and locations are: + +- `~/bigchaindb.log` +- `~/bigchaindb-errors.log` + +Log rotation is baked into Planetmint Server using Python's `logging` module. The logs for Planetmint Server are rotated when any of the above mentioned files exceeds 209715200 bytes (i.e. approximately 209 MB). + +For more information, see the docs about [the Planetmint Server configuration settings related to logging](../../installation/node-setup/configuration#log). + +## Tendermint Logging and Log Rotation + +Tendermint writes its logs to the files: + +- `tendermint.out.log` +- `tendermint.err.log` + +If you started Planetmint Server and Tendermint using Monit, as suggested by our guide on +[How to Set Up a Planetmint Network](../network-setup/network-setup), +then the logs will be written to `$HOME/.bigchaindb-monit/logs/`. + +Moreover, if you started Planetmint Server and Tendermint using Monit, +then Monit monitors the Tendermint log files. +Tendermint logs are rotated if any of the above mentioned log files exceeds 200 MB. diff --git a/docs/root/source/installation/appendices/ntp-notes.md b/docs/root/source/installation/appendices/ntp-notes.md new file mode 100644 index 0000000..5f92c72 --- /dev/null +++ b/docs/root/source/installation/appendices/ntp-notes.md @@ -0,0 +1,59 @@ + + +# Notes on NTP Daemon Setup + +There are several NTP daemons available, including: + +* The reference NTP daemon (`ntpd`) from ntp.org; see [their support website](http://support.ntp.org/bin/view/Support/WebHome) +* [chrony](https://chrony.tuxfamily.org/index.html) +* [OpenNTPD](http://www.openntpd.org/) +* Maybe [NTPsec](https://www.ntpsec.org/), once it's production-ready +* Maybe [Ntimed](http://nwtime.org/projects/ntimed/), once it's production-ready +* [More](https://en.wikipedia.org/wiki/Ntpd#Implementations) + +We suggest you run your NTP daemon in a mode which will tell your OS kernel to handle leap seconds in a particular way: the default NTP way, so that system clock adjustments are localized and not spread out across the minutes, hours, or days surrounding leap seconds (e.g. "slewing" or "smearing"). There's [a nice Red Hat Developer Blog post about the various options](https://developers.redhat.com/blog/2015/06/01/five-different-ways-handle-leap-seconds-ntp/). + +Use the default mode with `ntpd` and `chronyd`. For another NTP daemon, consult its documentation. + +It's tricky to make an NTP daemon setup secure. Always install the latest version and read the documentation about how to configure and run it securely. See the [notes on firewall setup](firewall-notes). + + +## Amazon Linux Instances + +If your Planetmint node is running on an Amazon Linux instance (i.e. a Linux instance packaged by Amazon, not Canonical, Red Hat, or someone else), then an NTP daemon should already be installed and configured. See the EC2 documentation on [Setting the Time for Your Linux Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html). + +That said, you should check _which_ NTP daemon is installed. Is it recent? Is it configured securely? + + +## The Ubuntu ntp Packages + +The [Ubuntu `ntp` packages](https://launchpad.net/ubuntu/+source/ntp) are based on the reference implementation of NTP. + +The following commands will uninstall the `ntp` and `ntpdate` packages, install the latest `ntp` package (which _might not be based on the latest ntpd code_), and start the NTP daemon (a local NTP server). (`ntpdate` is not reinstalled because it's [deprecated](https://askubuntu.com/questions/297560/ntpd-vs-ntpdate-pros-and-cons) and you shouldn't use it.) +```text +sudo apt-get --purge remove ntp ntpdate +sudo apt-get autoremove +sudo apt-get update +sudo apt-get install ntp +# That should start the NTP daemon too, but just to be sure: +sudo service ntp restart +``` + +You can check if `ntpd` is running using `sudo ntpq -p`. + +You may want to use different NTP time servers. You can change them by editing the NTP config file `/etc/ntp.conf`. + +Note: A server running an NTP daemon can be used by others for DRDoS amplification attacks. The above installation procedure should install a default NTP configuration file `/etc/ntp.conf` with the lines: +```text +restrict -4 default kod notrap nomodify nopeer noquery +restrict -6 default kod notrap nomodify nopeer noquery +``` + +Those lines should prevent the NTP daemon from being used in an attack. (The first line is for IPv4, the second for IPv6.) + +There are additional things you can do to make NTP more secure. See the [NTP Support Website](http://support.ntp.org/bin/view/Support/WebHome) for more details. diff --git a/docs/root/source/installation/commands-and-backend/backend.rst b/docs/root/source/installation/commands-and-backend/backend.rst new file mode 100644 index 0000000..1877d80 --- /dev/null +++ b/docs/root/source/installation/commands-and-backend/backend.rst @@ -0,0 +1,53 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +########################### +Database Backend Interfaces +########################### + +.. automodule:: bigchaindb.backend + :special-members: __init__ + + +Generic Interfaces +================== + +:mod:`bigchaindb.backend.connection` +------------------------------------ + +.. automodule:: bigchaindb.backend.connection + :special-members: __init__ + +:mod:`bigchaindb.backend.query` +------------------------------- +.. automodule:: bigchaindb.backend.query + +:mod:`bigchaindb.backend.schema` +-------------------------------- +.. automodule:: bigchaindb.backend.schema + +:mod:`bigchaindb.backend.utils` +------------------------------- +.. automodule:: bigchaindb.backend.utils + + +MongoDB Backend +=============== + +.. automodule:: bigchaindb.backend.localmongodb + :special-members: __init__ + +:mod:`bigchaindb.backend.localmongodb.connection` +------------------------------------------------- +.. automodule:: bigchaindb.backend.localmongodb.connection + +:mod:`bigchaindb.backend.localmongodb.query` +-------------------------------------------- +.. automodule:: bigchaindb.backend.localmongodb.query + +:mod:`bigchaindb.backend.localmongodb.schema` +--------------------------------------------- +.. automodule:: bigchaindb.backend.localmongodb.schema diff --git a/docs/root/source/installation/commands-and-backend/commands.rst b/docs/root/source/installation/commands-and-backend/commands.rst new file mode 100644 index 0000000..11b76e8 --- /dev/null +++ b/docs/root/source/installation/commands-and-backend/commands.rst @@ -0,0 +1,24 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +###################### +Command Line Interface +###################### + +.. automodule:: bigchaindb.commands + :special-members: __init__ + + +:mod:`bigchaindb.commands.bigchaindb` +------------------------------------- + +.. automodule:: bigchaindb.commands.bigchaindb + + +:mod:`bigchaindb.commands.utils` +-------------------------------- + +.. automodule:: bigchaindb.commands.utils diff --git a/docs/root/source/installation/commands-and-backend/index.rst b/docs/root/source/installation/commands-and-backend/index.rst new file mode 100644 index 0000000..bc2e927 --- /dev/null +++ b/docs/root/source/installation/commands-and-backend/index.rst @@ -0,0 +1,26 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Commands And Backend +==================== + +This section contains auto-generated documentation of various functions, classes and methods +in the Planetmint Server code, based on Python docstrings in the code itself. + +.. warning:: + + While we try to keep docstrings accurate, + if you want to know *for sure* what the code does, + then you have to read the code itself. + +.. toctree:: + :maxdepth: 1 + + commands + the-bigchaindb-class + backend + + \ No newline at end of file diff --git a/docs/root/source/installation/commands-and-backend/the-bigchaindb-class.rst b/docs/root/source/installation/commands-and-backend/the-bigchaindb-class.rst new file mode 100644 index 0000000..ac549c7 --- /dev/null +++ b/docs/root/source/installation/commands-and-backend/the-bigchaindb-class.rst @@ -0,0 +1,11 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +#################### +The Planetmint Class +#################### + +.. autoclass:: bigchaindb.Planetmint diff --git a/docs/root/source/installation/index.rst b/docs/root/source/installation/index.rst new file mode 100644 index 0000000..2efc18a --- /dev/null +++ b/docs/root/source/installation/index.rst @@ -0,0 +1,20 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Installation +============ + +You can install a single node to test out Planetmint, connect it to a network or setup a network of nodes. + +.. toctree:: + :maxdepth: 1 + + quickstart + node-setup/index + network-setup/index + api/index + commands-and-backend/index + appendices/index diff --git a/docs/root/source/installation/network-setup/bigchaindb-node-ansible.md b/docs/root/source/installation/network-setup/bigchaindb-node-ansible.md new file mode 100644 index 0000000..186be68 --- /dev/null +++ b/docs/root/source/installation/network-setup/bigchaindb-node-ansible.md @@ -0,0 +1,7 @@ +# Network of nodes with the Ansible script + +You can find one of the installation methods with Ansible on GitHub at: + +[Ansible script](https://github.com/bigchaindb/bigchaindb-node-ansible) + +It allows to install Planetmint, MongoDB, Tendermint, and python, and then connect nodes into a network. Current tested machine is Ubuntu 18.04. \ No newline at end of file diff --git a/docs/root/source/installation/network-setup/index.rst b/docs/root/source/installation/network-setup/index.rst new file mode 100644 index 0000000..ffc7659 --- /dev/null +++ b/docs/root/source/installation/network-setup/index.rst @@ -0,0 +1,19 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Network setup +============= + +There are several ways to setup a network. You can use the Kubernetes deployment template in this section, or use the Ansible solution in the Contributing section. Also, you can setup a single node on your machine and connect to an existing network. + +.. toctree:: + :maxdepth: 1 + + networks + network-setup + k8s-deployment-template/index + bigchaindb-node-ansible.md + \ No newline at end of file diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/architecture.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/architecture.rst new file mode 100644 index 0000000..f057ee8 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/architecture.rst @@ -0,0 +1,228 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Architecture of a Planetmint Node Running in a Kubernetes Cluster +================================================================= + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +If you deploy a Planetmint node into a Kubernetes cluster +as described in these docs, it will include: + +* NGINX, OpenResty, Planetmint, MongoDB and Tendermint + `Kubernetes Services `_. +* NGINX, OpenResty, Planetmint and MongoDB Monitoring Agent + `Kubernetes Deployments `_. +* MongoDB and Tendermint `Kubernetes StatefulSets `_. +* Third party services like `3scale `_, + `MongoDB Cloud Manager `_ and the + `Azure Operations Management Suite + `_. + + +.. _bigchaindb-node: + +Planetmint Node Diagram +----------------------- + +.. aafig:: + :aspect: 60 + :scale: 100 + :background: #rgb + :proportional: + + + + + +--------------------------------------------------------------------------------------------------------------------------------------+ + | | | | + | | | | + | | | | + | | | | + | | | | + | | | | + | "Planetmint API" | | "Tendermint P2P" | + | | | "Communication/" | + | | | "Public Key Exchange" | + | | | | + | | | | + | v v | + | | + | +------------------+ | + | |"NGINX Service" | | + | +-------+----------+ | + | | | + | v | + | | + | +------------------+ | + | | "NGINX" | | + | | "Deployment" | | + | | | | + | +-------+----------+ | + | | | + | | | + | | | + | v | + | | + | "443" +----------+ "26656/9986" | + | | "Rate" | | + | +---------------------------+"Limiting"+-----------------------+ | + | | | "Logic" | | | + | | +----+-----+ | | + | | | | | + | | | | | + | | | | | + | | | | | + | | | | | + | | "27017" | | | + | v | v | + | +-------------+ | +------------+ | + | |"HTTPS" | | +------------------> |"Tendermint"| | + | |"Termination"| | | "9986" |"Service" | "26656" | + | | | | | +-------+ | <----+ | + | +-----+-------+ | | | +------------+ | | + | | | | | | | + | | | | v v | + | | | | +------------+ +------------+ | + | | | | |"NGINX" | |"Tendermint"| | + | | | | |"Deployment"| |"Stateful" | | + | | | | |"Pub-Key-Ex"| |"Set" | | + | ^ | | +------------+ +------------+ | + | +-----+-------+ | | | + | "POST" |"Analyze" | "GET" | | | + | |"Request" | | | | + | +-----------+ +--------+ | | | + | | +-------------+ | | | | + | | | | | "Bi+directional, communication between" | + | | | | | "BigchainDBAPP) and Tendermint" | + | | | | | "BFT consensus Engine" | + | | | | | | + | v v | | | + | | | | + | +-------------+ +--------------+ +----+-------------------> +--------------+ | + | | "OpenResty" | | "Planetmint" | | | "MongoDB" | | + | | "Service" | | "Service" | | | "Service" | | + | | | +----->| | | +-------> | | | + | +------+------+ | +------+-------+ | | +------+-------+ | + | | | | | | | | + | | | | | | | | + | v | v | | v | + | +-------------+ | +-------------+ | | +----------+ | + | | | | | | <------------+ | |"MongoDB" | | + | |"OpenResty" | | | "Planetmint"| | |"Stateful"| | + | |"Deployment" | | | "Deployment"| | |"Set" | | + | | | | | | | +-----+----+ | + | | | | | +---------------------------+ | | + | | | | | | | | + | +-----+-------+ | +-------------+ | | + | | | | | + | | | | | + | v | | | + | +-----------+ | v | + | | "Auth" | | +------------+ | + | | "Logic" |----------+ |"MongoDB" | | + | | | |"Monitoring"| | + | | | |"Agent" | | + | +---+-------+ +-----+------+ | + | | | | + | | | | + | | | | + | | | | + | | | | + | | | | + +---------------+---------------------------------------------------------------------------------------+------------------------------+ + | | + | | + | | + v v + +------------------------------------+ +------------------------------------+ + | | | | + | | | | + | | | | + | "3Scale" | | "MongoDB Cloud" | + | | | | + | | | | + | | | | + +------------------------------------+ +------------------------------------+ + + + + +.. note:: + The arrows in the diagram represent the client-server communication. For + example, A-->B implies that A initiates the connection to B. + It does not represent the flow of data; the communication channel is always + fully duplex. + + +NGINX: Entrypoint and Gateway +----------------------------- + +We use an NGINX as HTTP proxy on port 443 (configurable) at the cloud +entrypoint for: + +#. Rate Limiting: We configure NGINX to allow only a certain number of requests + (configurable) which prevents DoS attacks. + +#. HTTPS Termination: The HTTPS connection does not carry through all the way + to Planetmint and terminates at NGINX for now. + +#. Request Routing: For HTTPS connections on port 443 (or the configured Planetmint public api port), + the connection is proxied to: + + #. OpenResty Service if it is a POST request. + #. Planetmint Service if it is a GET request. + + +We use an NGINX TCP proxy on port 27017 (configurable) at the cloud +entrypoint for: + +#. Rate Limiting: We configure NGINX to allow only a certain number of requests + (configurable) which prevents DoS attacks. + +#. Request Routing: For connections on port 27017 (or the configured MongoDB + public api port), the connection is proxied to the MongoDB Service. + + +OpenResty: API Management, Authentication and Authorization +----------------------------------------------------------- + +We use `OpenResty `_ to perform authorization checks +with 3scale using the ``app_id`` and ``app_key`` headers in the HTTP request. + +OpenResty is NGINX plus a bunch of other +`components `_. We primarily depend +on the LuaJIT compiler to execute the functions to authenticate the ``app_id`` +and ``app_key`` with the 3scale backend. + + +MongoDB: Standalone +------------------- + +We use MongoDB as the backend database for Planetmint. + +We achieve security by avoiding DoS attacks at the NGINX proxy layer and by +ensuring that MongoDB has TLS enabled for all its connections. + + +Tendermint: BFT consensus engine +-------------------------------- + +We use Tendermint as the backend consensus engine for BFT replication of Planetmint. +In a multi-node deployment, Tendermint nodes/peers communicate with each other via +the public ports exposed by the NGINX gateway. + +We use port **9986** (configurable) to allow tendermint nodes to access the public keys +of the peers and port **26656** (configurable) for the rest of the communications between +the peers. + diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/bigchaindb-network-on-kubernetes.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/bigchaindb-network-on-kubernetes.rst new file mode 100644 index 0000000..7f4f411 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/bigchaindb-network-on-kubernetes.rst @@ -0,0 +1,542 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _kubernetes-template-deploy-bigchaindb-network: + +Kubernetes Template: Deploying a Planetmint network +=================================================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +This page describes how to deploy a static Planetmint + Tendermint network. + +If you want to deploy a stand-alone Planetmint node in a Planetmint cluster, +or a stand-alone Planetmint node, +then see :doc:`the page about that `. + +We can use this guide to deploy a Planetmint network in the following scenarios: + +* Single Azure Kubernetes Site. +* Multiple Azure Kubernetes Sites (Geographically dispersed). + + +Terminology Used +---------------- + +``Planetmint node`` is a set of Kubernetes components that join together to +form a Planetmint single node. Please refer to the :doc:`architecture diagram ` +for more details. + +``Planetmint network`` will refer to a collection of nodes working together +to form a network. + + +Below, we refer to multiple files by their directory and filename, +such as ``bigchaindb/bigchaindb-ext-conn-svc.yaml``. Those files are located in the +`bigchaindb/bigchaindb repository on GitHub +`_ in the ``k8s/`` directory. +Make sure you're getting those files from the appropriate Git branch on +GitHub, i.e. the branch for the version of Planetmint that your Planetmint +cluster is using. + +.. note:: + + This deployment strategy is currently used for testing purposes only, + operated by a single stakeholder or tightly coupled stakeholders. + +.. note:: + + Currently, we only support a static set of participants in the network. + Once a Planetmint network is started with a certain number of validators + and a genesis file. Users cannot add new validator nodes dynamically. + You can track the progress of this funtionality on our + `github repository `_. + + +.. _pre-reqs-bdb-network: + +Prerequisites +------------- + +The deployment methodology is similar to one covered with :doc:`node-on-kubernetes`, but +we need to tweak some configurations depending on your choice of deployment. + +The operator needs to follow some consistent naming convention for all the components +covered :ref:`here `. + +Lets assume we are deploying a 4 node cluster, your naming conventions could look like this: + +.. code:: + + { + "MongoDB": [ + "mdb-instance-1", + "mdb-instance-2", + "mdb-instance-3", + "mdb-instance-4" + ], + "Planetmint": [ + "bdb-instance-1", + "bdb-instance-2", + "bdb-instance-3", + "bdb-instance-4" + ], + "NGINX": [ + "ngx-instance-1", + "ngx-instance-2", + "ngx-instance-3", + "ngx-instance-4" + ], + "OpenResty": [ + "openresty-instance-1", + "openresty-instance-2", + "openresty-instance-3", + "openresty-instance-4" + ], + "MongoDB_Monitoring_Agent": [ + "mdb-mon-instance-1", + "mdb-mon-instance-2", + "mdb-mon-instance-3", + "mdb-mon-instance-4" + ] + } + +.. note:: + + Blockchain Genesis ID and Time will be shared across all nodes. + +Edit config.yaml and secret.yaml +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Make N(number of nodes) copies of ``configuration/config-map.yaml`` and ``configuration/secret.yaml``. + +.. code:: text + + # For config-map.yaml + config-map-node-1.yaml + config-map-node-2.yaml + config-map-node-3.yaml + config-map-node-4.yaml + + # For secret.yaml + secret-node-1.yaml + secret-node-2.yaml + secret-node-3.yaml + secret-node-4.yaml + +Edit the data values as described in :doc:`this document `, based +on the naming convention described :ref:`above `. + +**Only for single site deployments**: Since all the configuration files use the +same ConfigMap and Secret Keys i.e. +``metadata.name -> vars, bdb-config and tendermint-config`` and +``metadata.name -> cloud-manager-credentials, mdb-certs, mdb-mon-certs, bdb-certs,`` +``https-certs, three-scale-credentials, ca-auth`` respectively, each file +will overwrite the configuration of the previously deployed one. +We want each node to have its own unique configurations. +One way to go about it is that, using the +:ref:`naming convention above ` we edit the ConfigMap and Secret keys. + +.. code:: text + + # For config-map-node-1.yaml + metadata.name: vars -> vars-node-1 + metadata.name: bdb-config -> bdb-config-node-1 + metadata.name: tendermint-config -> tendermint-config-node-1 + + # For secret-node-1.yaml + metadata.name: cloud-manager-credentials -> cloud-manager-credentials-node-1 + metadata.name: mdb-certs -> mdb-certs-node-1 + metadata.name: mdb-mon-certs -> mdb-mon-certs-node-1 + metadata.name: bdb-certs -> bdb-certs-node-1 + metadata.name: https-certs -> https-certs-node-1 + metadata.name: threescale-credentials -> threescale-credentials-node-1 + metadata.name: ca-auth -> ca-auth-node-1 + + # Repeat for the remaining files. + +Deploy all your configuration maps and secrets. + +.. code:: bash + + kubectl apply -f configuration/config-map-node-1.yaml + kubectl apply -f configuration/config-map-node-2.yaml + kubectl apply -f configuration/config-map-node-3.yaml + kubectl apply -f configuration/config-map-node-4.yaml + kubectl apply -f configuration/secret-node-1.yaml + kubectl apply -f configuration/secret-node-2.yaml + kubectl apply -f configuration/secret-node-3.yaml + kubectl apply -f configuration/secret-node-4.yaml + +.. note:: + + Similar to what we did, with config-map.yaml and secret.yaml i.e. indexing them + per node, we have to do the same for each Kubernetes component + i.e. Services, StorageClasses, PersistentVolumeClaims, StatefulSets, Deployments etc. + +.. code:: text + + # For Services + *-node-1-svc.yaml + *-node-2-svc.yaml + *-node-3-svc.yaml + *-node-4-svc.yaml + + # For StorageClasses + *-node-1-sc.yaml + *-node-2-sc.yaml + *-node-3-sc.yaml + *-node-4-sc.yaml + + # For PersistentVolumeClaims + *-node-1-pvc.yaml + *-node-2-pvc.yaml + *-node-3-pvc.yaml + *-node-4-pvc.yaml + + # For StatefulSets + *-node-1-ss.yaml + *-node-2-ss.yaml + *-node-3-ss.yaml + *-node-4-ss.yaml + + # For Deployments + *-node-1-dep.yaml + *-node-2-dep.yaml + *-node-3-dep.yaml + *-node-4-dep.yaml + + +.. _single-site-network: + +Single Site: Single Azure Kubernetes Cluster +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For the deployment of a Planetmint network under a single cluster, we need to replicate +the :doc:`deployment steps for each node ` N number of times, N being +the number of participants in the network. + +In our Kubernetes deployment template for a single Planetmint node, we covered the basic configurations +settings :ref:`here `. + +Since, we index the ConfigMap and Secret Keys for the single site deployment, we need to update +all the Kubernetes components to reflect the corresponding changes i.e. For each Kubernetes Service, +StatefulSet, PersistentVolumeClaim, Deployment, and StorageClass, we need to update the respective +`*.yaml` file and update the ConfigMapKeyRef.name OR secret.secretName. + +Example +""""""" + +Assuming we are deploying the MongoDB StatefulSet for Node 1. We need to update +the ``mongo-node-1-ss.yaml`` and update the corresponding ConfigMapKeyRef.name or secret.secretNames. + +.. code:: text + + ######################################################################## + # This YAML file desribes a StatefulSet with a service for running and # + # exposing a MongoDB instance. # + # It depends on the configdb and db k8s pvc. # + ######################################################################## + + apiVersion: apps/v1beta1 + kind: StatefulSet + metadata: + name: mdb-instance-0-ss + namespace: default + spec: + serviceName: mdb-instance-0 + replicas: 1 + template: + metadata: + name: mdb-instance-0-ss + labels: + app: mdb-instance-0-ss + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongodb + image: bigchaindb/mongodb:3.2 + imagePullPolicy: IfNotPresent + env: + - name: MONGODB_FQDN + valueFrom: + configMapKeyRef: + name: vars-1 # Changed from ``vars`` + key: mdb-instance-name + - name: MONGODB_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MONGODB_PORT + valueFrom: + configMapKeyRef: + name: vars-1 # Changed from ``vars`` + key: mongodb-backend-port + - name: STORAGE_ENGINE_CACHE_SIZE + valueFrom: + configMapKeyRef: + name: vars-1 # Changed from ``vars`` + key: storage-engine-cache-size + args: + - --mongodb-port + - $(MONGODB_PORT) + - --mongodb-key-file-path + - /etc/mongod/ssl/mdb-instance.pem + - --mongodb-ca-file-path + - /etc/mongod/ca/ca.pem + - --mongodb-crl-file-path + - /etc/mongod/ca/crl.pem + - --mongodb-fqdn + - $(MONGODB_FQDN) + - --mongodb-ip + - $(MONGODB_POD_IP) + - --storage-engine-cache-size + - $(STORAGE_ENGINE_CACHE_SIZE) + securityContext: + capabilities: + add: + - FOWNER + ports: + - containerPort: "" + protocol: TCP + name: mdb-api-port + volumeMounts: + - name: mdb-db + mountPath: /data/db + - name: mdb-configdb + mountPath: /data/configdb + - name: mdb-certs + mountPath: /etc/mongod/ssl/ + readOnly: true + - name: ca-auth + mountPath: /etc/mongod/ca/ + readOnly: true + resources: + limits: + cpu: 200m + memory: 5G + livenessProbe: + tcpSocket: + port: mdb-api-port + initialDelaySeconds: 15 + successThreshold: 1 + failureThreshold: 3 + periodSeconds: 15 + timeoutSeconds: 10 + restartPolicy: Always + volumes: + - name: mdb-db + persistentVolumeClaim: + claimName: mongo-db-claim-1 # Changed from ``mongo-db-claim`` + - name: mdb-configdb + persistentVolumeClaim: + claimName: mongo-configdb-claim-1 # Changed from ``mongo-configdb-claim`` + - name: mdb-certs + secret: + secretName: mdb-certs-1 # Changed from ``mdb-certs`` + defaultMode: 0400 + - name: ca-auth + secret: + secretName: ca-auth-1 # Changed from ``ca-auth`` + defaultMode: 0400 + +The above example is meant to be repeated for all the Kubernetes components of a Planetmint node. + +* ``nginx-http/nginx-http-node-X-svc.yaml`` or ``nginx-https/nginx-https-node-X-svc.yaml`` + +* ``nginx-http/nginx-http-node-X-dep.yaml`` or ``nginx-https/nginx-https-node-X-dep.yaml`` + +* ``mongodb/mongodb-node-X-svc.yaml`` + +* ``mongodb/mongodb-node-X-sc.yaml`` + +* ``mongodb/mongodb-node-X-pvc.yaml`` + +* ``mongodb/mongodb-node-X-ss.yaml`` + +* ``bigchaindb/bigchaindb-node-X-svc.yaml`` + +* ``bigchaindb/bigchaindb-node-X-sc.yaml`` + +* ``bigchaindb/bigchaindb-node-X-pvc.yaml`` + +* ``bigchaindb/bigchaindb-node-X-ss.yaml`` + +* ``nginx-openresty/nginx-openresty-node-X-svc.yaml`` + +* ``nginx-openresty/nginx-openresty-node-X-dep.yaml`` + + +Multi Site: Multiple Azure Kubernetes Clusters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For the multi site deployment of a Planetmint network with geographically dispersed +nodes, we need to replicate the :doc:`deployment steps for each node ` N number of times, +N being the number of participants in the network. + +The operator needs to follow a consistent naming convention which has :ref:`already +discussed in this document `. + +.. note:: + + Assuming we are using independent Kubernetes clusters, the ConfigMap and Secret Keys + do not need to be updated unlike :ref:`single-site-network`, and we also do not + need to update corresponding ConfigMap/Secret imports in the Kubernetes components. + + +Deploy Kubernetes Services +-------------------------- + +Deploy the following services for each node by following the naming convention +described :ref:`above `: + +* :ref:`Start the NGINX Service `. + +* :ref:`Assign DNS Name to the NGINX Public IP ` + +* :ref:`Start the MongoDB Kubernetes Service `. + +* :ref:`Start the Planetmint Kubernetes Service `. + +* :ref:`Start the OpenResty Kubernetes Service `. + + +Only for multi site deployments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We need to make sure that clusters are able +to talk to each other i.e. specifically the communication between the +Planetmint peers. Set up networking between the clusters using +`Kubernetes Services `_. + +Assuming we have a Planetmint instance ``bigchaindb-instance-1`` residing in Azure data center location ``westeurope`` and we +want to connect to ``bigchaindb-instance-2``, ``bigchaindb-instance-3``, and ``bigchaindb-instance-4`` located in Azure data centers +``eastus``, ``centralus`` and ``westus``, respectively. Unless you already have explicitly set up networking for +``bigchaindb-instance-1`` to communicate with ``bigchaindb-instance-2/3/4`` and +vice versa, we will have to add a Kubernetes Service in each cluster to accomplish this goal in order to set up a +Planetmint P2P network. +It is similar to ensuring that there is a ``CNAME`` record in the DNS +infrastructure to resolve ``bigchaindb-instance-X`` to the host where it is actually available. +We can do this in Kubernetes using a Kubernetes Service of ``type`` +``ExternalName``. + +* This configuration is located in the file ``bigchaindb/bigchaindb-ext-conn-svc.yaml``. + +* Set the name of the ``metadata.name`` to the host name of the Planetmint instance you are trying to connect to. + For instance if you are configuring this service on cluster with ``bigchaindb-instance-1`` then the ``metadata.name`` will + be ``bigchaindb-instance-2`` and vice versa. + +* Set ``spec.ports.port[0]`` to the ``tm-p2p-port`` from the ConfigMap for the other cluster. + +* Set ``spec.ports.port[1]`` to the ``tm-rpc-port`` from the ConfigMap for the other cluster. + +* Set ``spec.externalName`` to the FQDN mapped to NGINX Public IP of the cluster you are trying to connect to. + For more information about the FQDN please refer to: :ref:`Assign DNS name to NGINX Public + IP `. + +.. note:: + This operation needs to be replicated ``n-1`` times per node for a ``n`` node cluster, with the respective FQDNs + we need to communicate with. + + If you are not the system administrator of the cluster, you have to get in + touch with the system administrator/s of the other ``n-1`` clusters and + share with them your instance name (``bigchaindb-instance-name`` in the ConfigMap) + and the FQDN of the NGINX instance acting as Gateway(set in: :ref:`Assign DNS name to NGINX + Public IP `). + + +Start NGINX Kubernetes deployments +---------------------------------- + +Start the NGINX deployment that serves as a Gateway for each node by following the +naming convention described :ref:`above ` and referring to the following instructions: + +* :ref:`Start the NGINX Kubernetes Deployment `. + + +Deploy Kubernetes StorageClasses for MongoDB and Planetmint +------------------------------------------------------------ + +Deploy the following StorageClasses for each node by following the naming convention +described :ref:`above `: + +* :ref:`Create Kubernetes Storage Classes for MongoDB `. + +* :ref:`Create Kubernetes Storage Classes for Planetmint `. + + +Deploy Kubernetes PersistentVolumeClaims for MongoDB and Planetmint +-------------------------------------------------------------------- + +Deploy the following services for each node by following the naming convention +described :ref:`above `: + +* :ref:`Create Kubernetes Persistent Volume Claims for MongoDB `. + +* :ref:`Create Kubernetes Persistent Volume Claims for Planetmint ` + + +Deploy MongoDB Kubernetes StatefulSet +-------------------------------------- + +Deploy the MongoDB StatefulSet (standalone MongoDB) for each node by following the naming convention +described :ref:`above `: and referring to the following section: + +* :ref:`Start a Kubernetes StatefulSet for MongoDB `. + + +Configure Users and Access Control for MongoDB +---------------------------------------------- + +Configure users and access control for each MongoDB instance +in the network by referring to the following section: + +* :ref:`Configure Users and Access Control for MongoDB `. + + +Start Kubernetes StatefulSet for Planetmint +------------------------------------------- + +Start the Planetmint Kubernetes StatefulSet for each node by following the +naming convention described :ref:`above ` and referring to the following instructions: + +* :ref:`Start a Kubernetes Deployment for Planetmint `. + + +Start Kubernetes Deployment for MongoDB Monitoring Agent +--------------------------------------------------------- + +Start the MongoDB monitoring agent Kubernetes deployment for each node by following the +naming convention described :ref:`above ` and referring to the following instructions: + +* :ref:`Start a Kubernetes Deployment for MongoDB Monitoring Agent `. + + +Start Kubernetes Deployment for OpenResty +------------------------------------------ + +Start the OpenResty Kubernetes deployment for each node by following the +naming convention described :ref:`above ` and referring to the following instructions: + +* :ref:`Start a Kubernetes Deployment for OpenResty `. + + +Verify and Test +--------------- + +Verify and test your setup by referring to the following instructions: + +* :ref:`Verify the Planetmint Node Setup `. + diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/ca-installation.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/ca-installation.rst new file mode 100644 index 0000000..c8ca875 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/ca-installation.rst @@ -0,0 +1,101 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _how-to-set-up-a-self-signed-certificate-authority: + +How to Set Up a Self-Signed Certificate Authority +================================================= + +This page enumerates the steps *we* use to set up a self-signed certificate authority (CA). +This is something that only needs to be done once per Planetmint network, +by the organization managing the network, i.e. the CA is for the whole network. +We use Easy-RSA. + + +Step 1: Install & Configure Easy-RSA +------------------------------------ + +First create a directory for the CA and cd into it: + +.. code:: bash + + mkdir bdb-node-ca + + cd bdb-node-ca + +Then :ref:`install and configure Easy-RSA in that directory `. + + +Step 2: Create a Self-Signed CA +------------------------------- + +You can create a self-signed CA +by going to the ``bdb-node-ca/easy-rsa-3.0.1/easyrsa3`` directory and using: + +.. code:: bash + + ./easyrsa init-pki + + ./easyrsa build-ca + +You will also be asked to enter a PEM pass phrase (for encrypting the ``ca.key`` file). +Make sure to securely store that PEM pass phrase. +If you lose it, you won't be able to add or remove entities from your PKI infrastructure in the future. + +You will be prompted to enter the Distinguished Name (DN) information for this CA. +For each field, you can accept the default value [in brackets] by pressing Enter. + +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value ``ROOT-CA``. + +While ``Easy-RSA CA`` *is* a valid and acceptable Common Name, +you should probably enter a name based on the name of the managing organization, +e.g. ``Omega Ledger CA``. + +Tip: You can get help with the ``easyrsa`` command (and its subcommands) +by using the subcommand ``./easyrsa help`` + + +Step 3: Create an Intermediate CA +--------------------------------- + +TODO + +Step 4: Generate a Certificate Revocation List +---------------------------------------------- + +You can generate a Certificate Revocation List (CRL) using: + +.. code:: bash + + ./easyrsa gen-crl + +You will need to run this command every time you revoke a certificate. +The generated ``crl.pem`` needs to be uploaded to your infrastructure to +prevent the revoked certificate from being used again. + + +Step 5: Secure the CA +--------------------- + +The security of your infrastructure depends on the security of this CA. + +- Ensure that you restrict access to the CA and enable only legitimate and + required people to sign certificates and generate CRLs. + +- Restrict access to the machine where the CA is hosted. + +- Many certificate providers keep the CA offline and use a rotating + intermediate CA to sign and revoke certificates, to mitigate the risk of the + CA getting compromised. + +- In case you want to destroy the machine where you created the CA + (for example, if this was set up on a cloud provider instance), + you can backup the entire ``easyrsa`` directory + to secure storage. You can always restore it to a trusted instance again + during the times when you want to sign or revoke certificates. + Remember to backup the directory after every update. diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/client-tls-certificate.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/client-tls-certificate.rst new file mode 100644 index 0000000..09af7ce --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/client-tls-certificate.rst @@ -0,0 +1,111 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _how-to-generate-a-client-certificate-for-mongodb: + +How to Generate a Client Certificate for MongoDB +================================================ + +This page enumerates the steps *we* use to generate a client certificate to be +used by clients who want to connect to a TLS-secured MongoDB database. +We use Easy-RSA. + + +Step 1: Install and Configure Easy-RSA +-------------------------------------- + +First create a directory for the client certificate and cd into it: + +.. code:: bash + + mkdir client-cert + + cd client-cert + +Then :ref:`install and configure Easy-RSA in that directory `. + + +Step 2: Create the Client Private Key and CSR +--------------------------------------------- + +You can create the client private key and certificate signing request (CSR) +by going into the directory ``client-cert/easy-rsa-3.0.1/easyrsa3`` +and using: + +.. code:: bash + + ./easyrsa init-pki + + ./easyrsa gen-req bdb-instance-0 nopass + +You should change the Common Name (e.g. ``bdb-instance-0``) +to a value that reflects what the +client certificate is being used for, e.g. ``mdb-mon-instance-3`` or ``mdb-bak-instance-4``. (The final integer is specific to your Planetmint node in the Planetmint network.) + +You will be prompted to enter the Distinguished Name (DN) information for this certificate. For each field, you can accept the default value [in brackets] by pressing Enter. + +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value + ``Planetmint-Instance``, ``MongoDB-Mon-Instance`` or ``MongoDB-Backup-Instance`` + as appropriate. + +Aside: The ``nopass`` option means "do not encrypt the private key (default is encrypted)". You can get help with the ``easyrsa`` command (and its subcommands) +by using the subcommand ``./easyrsa help``. + +.. note:: + For more information about requirements for MongoDB client certificates, please consult the `official MongoDB + documentation `_. + + +Step 3: Get the Client Certificate Signed +----------------------------------------- + +The CSR file created in the previous step +should be located in ``pki/reqs/bdb-instance-0.req`` +(or whatever Common Name you used in the ``gen-req`` command above). +You need to send it to the organization managing the Planetmint network +so that they can use their CA +to sign the request. +(The managing organization should already have a self-signed CA.) + +If you are the admin of the managing organization's self-signed CA, +then you can import the CSR and use Easy-RSA to sign it. +Go to your ``bdb-node-ca/easy-rsa-3.0.1/easyrsa3/`` +directory and do something like: + +.. code:: bash + + ./easyrsa import-req /path/to/bdb-instance-0.req bdb-instance-0 + + ./easyrsa sign-req client bdb-instance-0 + +Once you have signed it, you can send the signed certificate +and the CA certificate back to the requestor. +The files are ``pki/issued/bdb-instance-0.crt`` and ``pki/ca.crt``. + + +Step 4: Generate the Consolidated Client PEM File +------------------------------------------------- + +.. note:: + This step can be skipped for Planetmint client certificate as Planetmint + uses the PyMongo driver, which accepts separate certificate and key files. + +MongoDB, MongoDB Backup Agent and MongoDB Monitoring Agent require a single, +consolidated file containing both the public and private keys. + +.. code:: bash + + cat /path/to/bdb-instance-0.crt /path/to/bdb-instance-0.key > bdb-instance-0.pem + + OR + + cat /path/to/mdb-mon-instance-0.crt /path/to/mdb-mon-instance-0.key > mdb-mon-instance-0.pem + + OR + + cat /path/to/mdb-bak-instance-0.crt /path/to/mdb-bak-instance-0.key > mdb-bak-instance-0.pem diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/cloud-manager.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/cloud-manager.rst new file mode 100644 index 0000000..12caf10 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/cloud-manager.rst @@ -0,0 +1,68 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _configure-mongodb-cloud-manager-for-monitoring: + +Configure MongoDB Cloud Manager for Monitoring +============================================== + +This document details the steps required to configure MongoDB Cloud Manager to +enable monitoring of data in a MongoDB Replica Set. + + +Configure MongoDB Cloud Manager for Monitoring Step by Step +----------------------------------------------------------- + + * Once the Monitoring Agent is up and running, open + `MongoDB Cloud Manager `_. + + * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud + Manager. + + * Select the group from the dropdown box on the page. + + * Go to Settings and add a ``Preferred Hostnames`` entry as + a regexp based on the ``mdb-instance-name`` of the nodes in your cluster. + It may take up to 5 mins till this setting takes effect. + You may refresh the browser window and verify whether the changes have + been saved or not. + + For example, for the nodes in a cluster that are named ``mdb-instance-0``, + ``mdb-instance-1`` and so on, a regex like ``^mdb-instance-[0-9]{1,2}$`` + is recommended. + + * Next, click the ``Deployment`` tab, and then the ``Manage Existing`` + button. + + * On the ``Import your deployment for monitoring`` page, enter the hostname + to be the same as the one set for ``mdb-instance-name`` in the global + ConfigMap for a node. + For example, if the ``mdb-instance-name`` is set to ``mdb-instance-0``, + enter ``mdb-instance-0`` as the value in this field. + + * Enter the port number as ``27017``, with no authentication. + + * If you have authentication enabled, select the option to enable + authentication and specify the authentication mechanism as per your + deployment. The default Planetmint Kubernetes deployment template currently + supports ``X.509 Client Certificate`` as the authentication mechanism. + + * If you have TLS enabled, select the option to enable TLS/SSL for MongoDB + connections, and click ``Continue``. This should already be selected for + you in case you selected ``X.509 Client Certificate`` above. + + * Wait a minute or two for the deployment to be found and then + click the ``Continue`` button again. + + * Verify that you see your process on the Cloud Manager UI. + It should look something like this: + + .. image:: ../../_static/mongodb_cloud_manager_1.png + + * Click ``Continue``. + + * Verify on the UI that data is being sent by the monitoring agent to the + Cloud Manager. It may take upto 5 minutes for data to appear on the UI. diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/easy-rsa.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/easy-rsa.rst new file mode 100644 index 0000000..21c664e --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/easy-rsa.rst @@ -0,0 +1,98 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _how-to-install-and-configure-easyrsa: + +How to Install & Configure Easy-RSA +=================================== + +We use +`Easy-RSA version 3 +`_, a +wrapper over complex ``openssl`` commands. +`Easy-RSA is available on GitHub `_ and licensed under GPLv2. + + +Step 1: Install Easy-RSA Dependencies +------------------------------------- + +The only dependency for Easy-RSA v3 is ``openssl``, +which is available from the ``openssl`` package on Ubuntu and other +Debian-based operating systems, i.e. you can install it using: + +.. code:: bash + + sudo apt-get update + + sudo apt-get install openssl + + +Step 2: Install Easy-RSA +------------------------ + +Make sure you're in the directory where you want Easy-RSA to live, +then download it and extract it within that directory: + +.. code:: bash + + wget https://github.com/OpenVPN/easy-rsa/archive/3.0.1.tar.gz + + tar xzvf 3.0.1.tar.gz + + rm 3.0.1.tar.gz + +There should now be a directory named ``easy-rsa-3.0.1`` +in your current directory. + + +Step 3: Customize the Easy-RSA Configuration +-------------------------------------------- + +We now create a config file named ``vars`` +by copying the existing ``vars.example`` file +and then editing it. +You should change the +country, province, city, org and email +to the correct values for your organisation. +(Note: The country, province, city, org and email are part of +the `Distinguished Name `_ (DN).) +The comments in the file explain what each of the variables mean. + +.. code:: bash + + cd easy-rsa-3.0.1/easyrsa3 + + cp vars.example vars + + echo 'set_var EASYRSA_DN "org"' >> vars + echo 'set_var EASYRSA_KEY_SIZE 4096' >> vars + + echo 'set_var EASYRSA_REQ_COUNTRY "DE"' >> vars + echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> vars + echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> vars + echo 'set_var EASYRSA_REQ_ORG "Planetmint GmbH"' >> vars + echo 'set_var EASYRSA_REQ_OU "IT"' >> vars + echo 'set_var EASYRSA_REQ_EMAIL "contact@ipdb.global"' >> vars + +Note: Later, when building a CA or generating a certificate signing request, you will be prompted to enter a value for the OU (or to accept the default). You should change the default OU from ``IT`` to one of the following, as appropriate: +``ROOT-CA``, +``MongoDB-Instance``, ``Planetmint-Instance``, ``MongoDB-Mon-Instance`` or +``MongoDB-Backup-Instance``. +To understand why, see `the MongoDB Manual `_. +There are reminders to do this in the relevant docs. + + +Step 4: Maybe Edit x509-types/server +------------------------------------ + +.. warning:: + + Only do this step if you are setting up a self-signed CA. + + Edit the file ``x509-types/server`` and change + ``extendedKeyUsage = serverAuth`` to + ``extendedKeyUsage = serverAuth,clientAuth``. + See `the MongoDB documentation about x.509 authentication `_ to understand why. diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/index.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/index.rst new file mode 100644 index 0000000..418732f --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/index.rst @@ -0,0 +1,48 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _kubernetes-deployment-template: + +Kubernetes Deployment Template +============================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +This section outlines a way to deploy a Planetmint node (or Planetmint network) +on Microsoft Azure using Kubernetes. +You may choose to use it as a template or reference for your own deployment, +but *we make no claim that it is suitable for your purposes*. +Feel free change things to suit your needs or preferences. + + +.. toctree:: + :maxdepth: 1 + + workflow + ca-installation + server-tls-certificate + client-tls-certificate + revoke-tls-certificate + template-kubernetes-azure + node-on-kubernetes + node-config-map-and-secrets + log-analytics + cloud-manager + easy-rsa + upgrade-on-kubernetes + bigchaindb-network-on-kubernetes + tectonic-azure + troubleshoot + architecture diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/log-analytics.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/log-analytics.rst new file mode 100644 index 0000000..b71dae5 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/log-analytics.rst @@ -0,0 +1,343 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Log Analytics on Azure +====================== + +This page describes how we use Microsoft Operations Management Suite (OMS) +to collect all logs from a Kubernetes cluster, +to search those logs, +and to set up email alerts based on log messages. +The :ref:`oms-k8s-references` section (below) contains links +to more detailed documentation. + +There are two steps: + +1. Setup: Create a log analytics OMS workspace + and a Containers solution under that workspace. +2. Deploy OMS agents to your Kubernetes cluster. + + +Step 1: Setup +------------- + +Step 1 can be done the web browser way or the command-line way. + + +The Web Browser Way +~~~~~~~~~~~~~~~~~~~ + +To create a new log analytics OMS workspace: + +1. Go to the Azure Portal in your web browser. +2. Click on **More services >** in the lower left corner of the Azure Portal. +3. Type "log analytics" or similar. +4. Select **Log Analytics** from the list of options. +5. Click on **+ Add** to add a new log analytics OMS workspace. +6. Give answers to the questions. You can call the OMS workspace anything, + but use the same resource group and location as your Kubernetes cluster. + The free option will suffice, but of course you can also use a paid one. + +To add a "Containers solution" to that new workspace: + +1. In Azure Portal, in the Log Analytics section, click the name of the new workspace +2. Click **OMS Workspace**. +3. Click **OMS Portal**. It should launch the OMS Portal in a new tab. +4. Click the **Solutions Gallery** tile. +5. Click the **Containers** tile. +6. Click **Add**. + + +The Command-Line Way +~~~~~~~~~~~~~~~~~~~~ + +We'll assume your Kubernetes cluster has a resource +group named: + +* ``resource_group`` + +and the workspace we'll create will be named: + +* ``work_space`` + +If you feel creative you may replace these names by more interesting ones. + +.. code-block:: bash + + $ az group deployment create --debug \ + --resource-group resource_group \ + --name "Microsoft.LogAnalyticsOMS" \ + --template-file log_analytics_oms.json \ + --parameters @log_analytics_oms.parameters.json + +An example of a simple template file (``--template-file``): + +.. code-block:: json + + { + "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sku": { + "type": "String" + }, + "workspaceName": { + "type": "String" + }, + "solutionType": { + "type": "String" + }, + "resources": [ + { + "apiVersion": "2015-03-20", + "type": "Microsoft.OperationalInsights/workspaces", + "name": "[parameters('workspaceName')]", + "location": "[resourceGroup().location]", + "properties": { + "sku": { + "name": "[parameters('sku')]" + } + }, + "resources": [ + { + "apiVersion": "2015-11-01-preview", + "location": "[resourceGroup().location]", + "name": "[Concat(parameters('solutionType'), '(', parameters('workspaceName'), ')')]", + "type": "Microsoft.OperationsManagement/solutions", + "id": "[Concat(resourceGroup().id, '/providers/Microsoft.OperationsManagement/solutions/', parameters('solutionType'), '(', parameters('workspaceName'), ')')]", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]" + ], + "properties": { + "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]" + }, + "plan": { + "publisher": "Microsoft", + "product": "[Concat('OMSGallery/', parameters('solutionType'))]", + "name": "[Concat(parameters('solutionType'), '(', parameters('workspaceName'), ')')]", + "promotionCode": "" + } + } + ] + } + ] + } + } + +An example of the associated parameter file (``--parameters``): + +.. code-block:: json + + { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sku": { + "value": "Free" + }, + "workspaceName": { + "value": "work_space" + }, + "solutionType": { + "value": "Containers" + } + } + } + + +Step 2: Deploy the OMS Agents +----------------------------- + +To deploy an OMS agent, two important pieces of information are needed: + +1. workspace id +2. workspace key + +You can obtain the workspace id using: + +.. code-block:: bash + + $ az resource show \ + --resource-group resource_group + --resource-type Microsoft.OperationalInsights/workspaces + --name work_space \ + | grep customerId + "customerId": "12345678-1234-1234-1234-123456789012", + +Until we figure out a way to obtain the *workspace key* via the command line, +you can get it via the OMS Portal. +To get to the OMS Portal, go to the Azure Portal and click on: + +Resource Groups > (Your Kubernetes cluster's resource group) > Log analytics (OMS) > (Name of the only item listed) > OMS Workspace > OMS Portal + +(Let us know if you find a faster way.) +Then see `Microsoft's instructions to obtain your workspace ID and key +`_ (via the OMS Portal). + +Once you have the workspace id and key, you can include them in the following +YAML file (:download:`oms-daemonset.yaml +<../../../../../../k8s/logging-and-monitoring/oms-daemonset.yaml>`): + +.. code-block:: yaml + + # oms-daemonset.yaml + apiVersion: extensions/v1beta1 + kind: DaemonSet + metadata: + name: omsagent + spec: + template: + metadata: + labels: + app: omsagent + spec: + containers: + - env: + - name: WSID + value: + - name: KEY + value: + image: microsoft/oms + name: omsagent + ports: + - containerPort: 25225 + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + +To deploy the OMS agents (one per Kubernetes node, i.e. one per computer), +simply run the following command: + +.. code-block:: bash + + $ kubectl create -f oms-daemonset.yaml + + +Search the OMS Logs +------------------- + +OMS should now be getting, storing and indexing all the logs +from all the containers in your Kubernetes cluster. +You can search the OMS logs from the Azure Portal +or the OMS Portal, but at the time of writing, +there was more functionality in the OMS Portal +(e.g. the ability to create an Alert based on a search). + +There are instructions to get to the OMS Portal above. +Once you're in the OMS Portal, click on **Log Search** +and enter a query. +Here are some example queries: + +All logging messages containing the strings "critical" or "error" (not case-sensitive): + +``Type=ContainerLog (critical OR error)`` + +.. note:: + + You can filter the results even more by clicking on things in the left sidebar. + For OMS Log Search syntax help, see the + `Log Analytics search reference `_. + +All logging messages containing the string "error" but not "404": + +``Type=ContainerLog error NOT(404)`` + +All logging messages containing the string "critical" but not "CriticalAddonsOnly": + +``Type=ContainerLog critical NOT(CriticalAddonsOnly)`` + +All logging messages from containers running the Docker image bigchaindb/nginx_3scale:1.3, containing the string "GET" but not the strings "Go-http-client" or "runscope" (where those exclusions filter out tests by Kubernetes and Runscope): + +``Type=ContainerLog Image="bigchaindb/nginx_3scale:1.3" GET NOT("Go-http-client") NOT(runscope)`` + +.. note:: + + We wrote a small Python 3 script to analyze the logs found by the above NGINX search. + It's in ``k8s/logging-and-monitoring/analyze.py``. The docsting at the top + of the script explains how to use it. + + +Create an Email Alert +--------------------- + +Once you're satisfied with an OMS Log Search query string, +click the **🔔 Alert** icon in the top menu, +fill in the form, +and click **Save** when you're done. + + +Some Useful Management Tasks +---------------------------- +List workspaces: + +.. code-block:: bash + + $ az resource list \ + --resource-group resource_group \ + --resource-type Microsoft.OperationalInsights/workspaces + +List solutions: + +.. code-block:: bash + + $ az resource list \ + --resource-group resource_group \ + --resource-type Microsoft.OperationsManagement/solutions + +Delete the containers solution: + +.. code-block:: bash + + $ az group deployment delete --debug \ + --resource-group resource_group \ + --name Microsoft.ContainersOMS + +.. code-block:: bash + + $ az resource delete \ + --resource-group resource_group \ + --resource-type Microsoft.OperationsManagement/solutions \ + --name "Containers(work_space)" + +Delete the workspace: + +.. code-block:: bash + + $ az group deployment delete --debug \ + --resource-group resource_group \ + --name Microsoft.LogAnalyticsOMS + +.. code-block:: bash + + $ az resource delete \ + --resource-group resource_group \ + --resource-type Microsoft.OperationalInsights/workspaces \ + --name work_space + + +.. _oms-k8s-references: + +References +---------- + +* `Monitor an Azure Container Service cluster with Microsoft Operations Management Suite (OMS) `_ +* `Manage Log Analytics using Azure Resource Manager templates `_ +* `azure commands for deployments `_ + (``az group deployment``) +* `Understand the structure and syntax of Azure Resource Manager templates `_ +* `Kubernetes DaemonSet`_ + + + +.. _Azure Resource Manager templates: https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates +.. _Kubernetes DaemonSet: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/node-config-map-and-secrets.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/node-config-map-and-secrets.rst new file mode 100644 index 0000000..0b09e23 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/node-config-map-and-secrets.rst @@ -0,0 +1,124 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _how-to-configure-a-bigchaindb-node: + +How to Configure a Planetmint Node +================================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +This page outlines the steps to set a bunch of configuration settings +in your Planetmint node. +They are pushed to the Kubernetes cluster in two files, +named ``config-map.yaml`` (a set of ConfigMaps) +and ``secret.yaml`` (a set of Secrets). +They are stored in the Kubernetes cluster's key-value store (etcd). + +Make sure you did the first four operations listed in the section titled +:ref:`things-each-node-operator-must-do`. + + +Edit vars +--------- + +This file is located at: ``k8s/scripts/vars`` and edit +the configuration parameters. +That file already contains many comments to help you +understand each data value, but we make some additional +remarks on some of the values below. + + +vars.NODE_FQDN +~~~~~~~~~~~~~~~ +FQDN for your Planetmint node. This is the domain name +used to query and access your Planetmint node. More information can be +found in our :ref:`Kubernetes template overview guide `. + + +vars.SECRET_TOKEN +~~~~~~~~~~~~~~~~~ +This parameter is specific to your Planetmint node and is used for +authentication and authorization of requests to your Planetmint node. +More information can be found in our :ref:`Kubernetes template overview guide `. + + +vars.HTTPS_CERT_KEY_FILE_NAME +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Absolute path of the HTTPS certificate chain of your domain. +More information can be found in our :ref:`Kubernetes template overview guide `. + + +vars.HTTPS_CERT_CHAIN_FILE_NAME +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Absolute path of the HTTPS certificate key of your domain. +More information can be found in our :ref:`Kubernetes template overview guide `. + + +vars.MDB_ADMIN_USER and vars.MDB_ADMIN_PASSWORD +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +MongoDB admin user credentials, username and password. +This user is created on the *admin* database with the authorization to create other users. + + +vars.BDB_PERSISTENT_PEERS, BDB_VALIDATORS, BDB_VALIDATORS_POWERS, BDB_GENESIS_TIME and BDB_CHAIN_ID +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +These parameters are shared across the Planetmint network. More information about the generation +of these parameters can be found at :ref:`generate-the-blockchain-id-and-genesis-time`. + + +vars.NODE_DNS_SERVER +^^^^^^^^^^^^^^^^^^^^ +IP of Kubernetes service(kube-dns), can be retrieved using +using CLI(kubectl) or k8s dashboard. This parameter is used by the Nginx gateway instance +to resolve the hostnames of all the services running in the Kubernetes cluster. + +.. code:: + + # retrieval via commandline. + $ kubectl get services --namespace=kube-system -l k8s-app=kube-dns + + +.. _generate-config: + +Generate configuration +~~~~~~~~~~~~~~~~~~~~~~ +After populating the ``k8s/scripts/vars`` file, we need to generate +all the configuration required for the Planetmint node, for that purpose +we need to execute ``k8s/scripts/generate_configs.sh`` script. + +.. code:: + + $ bash generate_configs.sh + +.. Note:: + During execution the script will prompt the user for some inputs. + +After successful execution, this routine will generate ``config-map.yaml`` and +``secret.yaml`` under ``k8s/scripts``. + +.. _deploy-config-map-and-secret: + +Deploy Your config-map.yaml and secret.yaml +------------------------------------------- + +You can deploy your edited ``config-map.yaml`` and ``secret.yaml`` +files to your Kubernetes cluster using the commands: + +.. code:: bash + + $ kubectl apply -f config-map.yaml + + $ kubectl apply -f secret.yaml diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/node-on-kubernetes.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/node-on-kubernetes.rst new file mode 100644 index 0000000..15d7ca3 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/node-on-kubernetes.rst @@ -0,0 +1,769 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _kubernetes-template-deploy-a-single-bigchaindb-node: + +Kubernetes Template: Deploy a Single Planetmint Node +==================================================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +This page describes how to deploy a Planetmint node +using `Kubernetes `_. +It assumes you already have a running Kubernetes cluster. + +Below, we refer to many files by their directory and filename, +such as ``configuration/config-map.yaml``. Those files are files in the +`bigchaindb/bigchaindb repository on GitHub `_ +in the ``k8s/`` directory. +Make sure you're getting those files from the appropriate Git branch on +GitHub, i.e. the branch for the version of Planetmint that your Planetmint +cluster is using. + + +Step 1: Install and Configure kubectl +------------------------------------- + +kubectl is the Kubernetes CLI. +If you don't already have it installed, +then see the `Kubernetes docs to install it +`_. + +The default location of the kubectl configuration file is ``~/.kube/config``. +If you don't have that file, then you need to get it. + +**Azure.** If you deployed your Kubernetes cluster on Azure +using the Azure CLI 2.0 (as per :doc:`our template +<../k8s-deployment-template/template-kubernetes-azure>`), +then you can get the ``~/.kube/config`` file using: + +.. code:: bash + + $ az acs kubernetes get-credentials \ + --resource-group \ + --name + +If it asks for a password (to unlock the SSH key) +and you enter the correct password, +but you get an error message, +then try adding ``--ssh-key-file ~/.ssh/`` +to the above command (i.e. the path to the private key). + +.. note:: + + **About kubectl contexts.** You might manage several + Kubernetes clusters. To make it easy to switch from one to another, + kubectl has a notion of "contexts," e.g. the context for cluster 1 or + the context for cluster 2. To find out the current context, do: + + .. code:: bash + + $ kubectl config view + + and then look for the ``current-context`` in the output. + The output also lists all clusters, contexts and users. + (You might have only one of each.) + You can switch to a different context using: + + .. code:: bash + + $ kubectl config use-context + + You can also switch to a different context for just one command + by inserting ``--context `` into any kubectl command. + For example: + + .. code:: bash + + $ kubectl get pods + + will get a list of the pods in the Kubernetes cluster associated + with the context named ``k8s-bdb-test-cluster-0``. + +Step 2: Connect to Your Kubernetes Cluster's Web UI (Optional) +--------------------------------------------------------------- + +You can connect to your cluster's +`Kubernetes Dashboard `_ +(also called the Web UI) using: + +.. code:: bash + + $ kubectl proxy -p 8001 + + or + + $ az acs kubernetes browse -g [Resource Group] -n [Container service instance name] --ssh-key-file /path/to/privateKey + +or, if you prefer to be explicit about the context (explained above): + +.. code:: bash + + $ kubectl proxy -p 8001 + +The output should be something like ``Starting to serve on 127.0.0.1:8001``. +That means you can visit the dashboard in your web browser at +`http://127.0.0.1:8001/ui `_. + +.. note:: + + **Known Issue:** If you are having accessing the UI i.e. + accessing `http://127.0.0.1:8001/ui `_ + in your browser returns a blank page and is redirected to + `http://127.0.0.1:8001/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy + `_ + , you can access the UI by adding a **/** at the end of the redirected URL i.e. + `http://127.0.0.1:8001/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy/ + `_ + + +Step 3: Configure Your Planetmint Node +-------------------------------------- + +See the page titled :ref:`how-to-configure-a-bigchaindb-node`. + + +.. _start-the-nginx-service: + +Step 4: Start the NGINX Service +------------------------------- + + * This will will give us a public IP for the cluster. + + * Once you complete this step, you might need to wait up to 10 mins for the + public IP to be assigned. + + * You have the option to use vanilla NGINX without HTTPS support or an + NGINX with HTTPS support. + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl apply -f nginx-https/nginx-https-svc.yaml + + OR + + $ kubectl apply -f nginx-http/nginx-http-svc.yaml + + +.. _assign-dns-name-to-nginx-public-ip: + +Step 5: Assign DNS Name to the NGINX Public IP +---------------------------------------------- + + * This step is required only if you are planning to set up multiple + `Planetmint nodes + `_ or are using + HTTPS certificates tied to a domain. + + * The following command can help you find out if the NGINX service started + above has been assigned a public IP or external IP address: + + .. code:: bash + + $ kubectl get svc -w + + * Once a public IP is assigned, you can map it to + a DNS name. + We usually assign ``bdb-test-node-0``, ``bdb-test-node-1`` and + so on in our documentation. + Let's assume that we assign the unique name of ``bdb-test-node-0`` here. + + +**Set up DNS mapping in Azure.** +Select the current Azure resource group and look for the ``Public IP`` +resource. You should see at least 2 entries there - one for the Kubernetes +master and the other for the NGINX instance. You may have to ``Refresh`` the +Azure web page listing the resources in a resource group for the latest +changes to be reflected. +Select the ``Public IP`` resource that is attached to your service (it should +have the Azure DNS prefix name along with a long random string, without the +``master-ip`` string), select ``Configuration``, add the DNS assigned above +(for example, ``bdb-test-node-0``), click ``Save``, and wait for the +changes to be applied. + +To verify the DNS setting is operational, you can run ``nslookup `` from your local Linux shell. + +This will ensure that when you scale to different geographical zones, other Tendermint +nodes in the network can reach this instance. + + +.. _start-the-mongodb-kubernetes-service: + +Step 6: Start the MongoDB Kubernetes Service +-------------------------------------------- + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl apply -f mongodb/mongo-svc.yaml + + +.. _start-the-bigchaindb-kubernetes-service: + +Step 7: Start the Planetmint Kubernetes Service +----------------------------------------------- + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl apply -f bigchaindb/bigchaindb-svc.yaml + + +.. _start-the-openresty-kubernetes-service: + +Step 8(Optional): Start the OpenResty Kubernetes Service +--------------------------------------------------------- + + * Start the Kubernetes Service: + + .. code:: bash + + $ kubectl apply -f nginx-openresty/nginx-openresty-svc.yaml + + +.. _start-the-nginx-deployment: + +Step 9: Start the NGINX Kubernetes Deployment +---------------------------------------------- + + * NGINX is used as a proxy to the Planetmint, Tendermint and MongoDB instances in + the node. It proxies HTTP/HTTPS requests on the ``node-frontend-port`` + to the corresponding OpenResty(if 3scale enabled) or Planetmint backend, TCP connections + on ``mongodb-frontend-port``, ``tm-p2p-port`` and ``tm-pub-key-access`` + to MongoDB and Tendermint respectively. + + * This configuration is located in the file + ``nginx-https/nginx-https-dep.yaml`` or ``nginx-http/nginx-http-dep.yaml``. + + * Start the Kubernetes Deployment: + + .. code:: bash + + $ kubectl apply -f nginx-https/nginx-https-dep.yaml + + OR + + $ kubectl apaply -f nginx-http/nginx-http-dep.yaml + + +.. _create-kubernetes-storage-class-mdb: + +Step 10: Create Kubernetes Storage Classes for MongoDB +------------------------------------------------------ + +MongoDB needs somewhere to store its data persistently, +outside the container where MongoDB is running. +Our MongoDB Docker container +(based on the official MongoDB Docker container) +exports two volume mounts with correct +permissions from inside the container: + +* The directory where the MongoDB instance stores its data: ``/data/db``. + There's more explanation in the MongoDB docs about `storage.dbpath `_. + +* The directory where the MongoDB instance stores the metadata for a sharded + cluster: ``/data/configdb/``. + There's more explanation in the MongoDB docs about `sharding.configDB `_. + +Explaining how Kubernetes handles persistent volumes, +and the associated terminology, +is beyond the scope of this documentation; +see `the Kubernetes docs about persistent volumes +`_. + +The first thing to do is create the Kubernetes storage classes. + +**Set up Storage Classes in Azure.** +First, you need an Azure storage account. +If you deployed your Kubernetes cluster on Azure +using the Azure CLI 2.0 +(as per :doc:`our template <../k8s-deployment-template/template-kubernetes-azure>`), +then the `az acs create` command already created a +storage account in the same location and resource group +as your Kubernetes cluster. +Both should have the same "storage account SKU": ``Standard_LRS``. +Standard storage is lower-cost and lower-performance. +It uses hard disk drives (HDD). +LRS means locally-redundant storage: three replicas +in the same data center. +Premium storage is higher-cost and higher-performance. +It uses solid state drives (SSD). + +We recommend using Premium storage with our Kubernetes deployment template. +Create a `storage account `_ +for Premium storage and associate it with your Azure resource group. +For future reference, the command to create a storage account is +`az storage account create `_. + +.. note:: + Please refer to `Azure documentation `_ + for the list of VMs that are supported by Premium Storage. + +The Kubernetes template for configuration of the MongoDB Storage Class is located in the +file ``mongodb/mongo-sc.yaml``. + +You may have to update the ``parameters.location`` field in the file to +specify the location you are using in Azure. + +If you want to use a custom storage account with the Storage Class, you +can also update `parameters.storageAccount` and provide the Azure storage +account name. + +Create the required storage classes using: + +.. code:: bash + + $ kubectl apply -f mongodb/mongo-sc.yaml + + +You can check if it worked using ``kubectl get storageclasses``. + + +.. _create-kubernetes-persistent-volume-claim-mdb: + +Step 11: Create Kubernetes Persistent Volume Claims for MongoDB +--------------------------------------------------------------- + +Next, you will create two PersistentVolumeClaim objects ``mongo-db-claim`` and +``mongo-configdb-claim``. + +This configuration is located in the file ``mongodb/mongo-pvc.yaml``. + +Note how there's no explicit mention of Azure, AWS or whatever. +``ReadWriteOnce`` (RWO) means the volume can be mounted as +read-write by a single Kubernetes node. +(``ReadWriteOnce`` is the *only* access mode supported +by AzureDisk.) +``storage: 20Gi`` means the volume has a size of 20 +`gibibytes `_. + +You may want to update the ``spec.resources.requests.storage`` field in both +the files to specify a different disk size. + +Create the required Persistent Volume Claims using: + +.. code:: bash + + $ kubectl apply -f mongodb/mongo-pvc.yaml + + +You can check its status using: ``kubectl get pvc -w`` + +Initially, the status of persistent volume claims might be "Pending" +but it should become "Bound" fairly quickly. + +.. note:: + The default Reclaim Policy for dynamically created persistent volumes is ``Delete`` + which means the PV and its associated Azure storage resource will be automatically + deleted on deletion of PVC or PV. In order to prevent this from happening do + the following steps to change default reclaim policy of dyanmically created PVs + from ``Delete`` to ``Retain`` + + * Run the following command to list existing PVs + + .. Code:: bash + + $ kubectl get pv + + * Run the following command to update a PV's reclaim policy to + + .. Code:: bash + + $ kubectl patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + + For notes on recreating a private volume form a released Azure disk resource consult + :doc:`the page about cluster troubleshooting <../k8s-deployment-template/troubleshoot>`. + +.. _start-kubernetes-stateful-set-mongodb: + +Step 12: Start a Kubernetes StatefulSet for MongoDB +--------------------------------------------------- + + * Create the MongoDB StatefulSet using: + + .. code:: bash + + $ kubectl apply -f mongodb/mongo-ss.yaml + + * It might take up to 10 minutes for the disks, specified in the Persistent + Volume Claims above, to be created and attached to the pod. + The UI might show that the pod has errored with the message + "timeout expired waiting for volumes to attach/mount". Use the CLI below + to check the status of the pod in this case, instead of the UI. + This happens due to a bug in Azure ACS. + + .. code:: bash + + $ kubectl get pods -w + + +.. _configure-users-and-access-control-mongodb: + +Step 13: Configure Users and Access Control for MongoDB +------------------------------------------------------- + + * In this step, you will create a user on MongoDB with authorization + to create more users and assign roles to it. We will also create + MongoDB client users for Planetmint and MongoDB Monitoring agent(Optional). + + .. code:: bash + + $ kubectl apply -f mongodb/configure_mdb.sh + + +.. _create-kubernetes-storage-class: + +Step 14: Create Kubernetes Storage Classes for Planetmint +---------------------------------------------------------- + +Planetmint needs somewhere to store Tendermint data persistently, Tendermint uses +LevelDB as the persistent storage layer. + +The Kubernetes template for configuration of Storage Class is located in the +file ``bigchaindb/bigchaindb-sc.yaml``. + +Details about how to create a Azure Storage account and how Kubernetes Storage Class works +are already covered in this document: :ref:`create-kubernetes-storage-class-mdb`. + +Create the required storage classes using: + +.. code:: bash + + $ kubectl apply -f bigchaindb/bigchaindb-sc.yaml + + +You can check if it worked using ``kubectl get storageclasses``. + +.. _create-kubernetes-persistent-volume-claim: + +Step 15: Create Kubernetes Persistent Volume Claims for Planetmint +------------------------------------------------------------------ + +Next, you will create two PersistentVolumeClaim objects ``tendermint-db-claim`` and +``tendermint-config-db-claim``. + +This configuration is located in the file ``bigchaindb/bigchaindb-pvc.yaml``. + +Details about Kubernetes Persistent Volumes, Persistent Volume Claims +and how they work with Azure are already covered in this +document: :ref:`create-kubernetes-persistent-volume-claim-mdb`. + +Create the required Persistent Volume Claims using: + +.. code:: bash + + $ kubectl apply -f bigchaindb/bigchaindb-pvc.yaml + +You can check its status using: + +.. code:: + + kubectl get pvc -w + + +.. _start-kubernetes-stateful-set-bdb: + +Step 16: Start a Kubernetes StatefulSet for Planetmint +------------------------------------------------------ + + * This configuration is located in the file ``bigchaindb/bigchaindb-ss.yaml``. + + * Set the ``spec.serviceName`` to the value set in ``bdb-instance-name`` in + the ConfigMap. + For example, if the value set in the ``bdb-instance-name`` + is ``bdb-instance-0``, set the field to ``tm-instance-0``. + + * Set ``metadata.name``, ``spec.template.metadata.name`` and + ``spec.template.metadata.labels.app`` to the value set in + ``bdb-instance-name`` in the ConfigMap, followed by + ``-ss``. + For example, if the value set in the + ``bdb-instance-name`` is ``bdb-instance-0``, set the fields to the value + ``bdb-insance-0-ss``. + + * As we gain more experience running Tendermint in testing and production, we + will tweak the ``resources.limits.cpu`` and ``resources.limits.memory``. + + * Create the Planetmint StatefulSet using: + + .. code:: bash + + $ kubectl apply -f bigchaindb/bigchaindb-ss.yaml + + .. code:: bash + + $ kubectl get pods -w + + +.. _start-kubernetes-deployment-for-mdb-mon-agent: + +Step 17(Optional): Start a Kubernetes Deployment for MongoDB Monitoring Agent +------------------------------------------------------------------------------ + + * This configuration is located in the file + ``mongodb-monitoring-agent/mongo-mon-dep.yaml``. + + * Set ``metadata.name``, ``spec.template.metadata.name`` and + ``spec.template.metadata.labels.app`` to the value set in + ``mdb-mon-instance-name`` in the ConfigMap, followed by + ``-dep``. + For example, if the value set in the + ``mdb-mon-instance-name`` is ``mdb-mon-instance-0``, set the fields to the + value ``mdb-mon-instance-0-dep``. + + * The configuration uses the following values set in the Secret: + + - ``mdb-mon-certs`` + - ``ca-auth`` + - ``cloud-manager-credentials`` + + * Start the Kubernetes Deployment using: + + .. code:: bash + + $ kubectl apply -f mongodb-monitoring-agent/mongo-mon-dep.yaml + + +.. _start-kubernetes-deployment-openresty: + +Step 18(Optional): Start a Kubernetes Deployment for OpenResty +-------------------------------------------------------------- + + * This configuration is located in the file + ``nginx-openresty/nginx-openresty-dep.yaml``. + + * Set ``metadata.name`` and ``spec.template.metadata.labels.app`` to the + value set in ``openresty-instance-name`` in the ConfigMap, followed by + ``-dep``. + For example, if the value set in the + ``openresty-instance-name`` is ``openresty-instance-0``, set the fields to + the value ``openresty-instance-0-dep``. + + * Set the port to be exposed from the pod in the + ``spec.containers[0].ports`` section. We currently expose the port at + which OpenResty is listening for requests, ``openresty-backend-port`` in + the above ConfigMap. + + * The configuration uses the following values set in the Secret: + + - ``threescale-credentials`` + + * The configuration uses the following values set in the ConfigMap: + + - ``node-dns-server-ip`` + - ``openresty-backend-port`` + - ``ngx-bdb-instance-name`` + - ``bigchaindb-api-port`` + + * Create the OpenResty Deployment using: + + .. code:: bash + + $ kubectl apply -f nginx-openresty/nginx-openresty-dep.yaml + + + * You can check its status using the command ``kubectl get deployments -w`` + + +Step 19(Optional): Configure the MongoDB Cloud Manager +------------------------------------------------------ + +Refer to the +:doc:`documentation <../k8s-deployment-template/cloud-manager>` +for details on how to configure the MongoDB Cloud Manager to enable +monitoring and backup. + + +Step 20(Optional): Only for multi site deployments(Geographically dispersed) +---------------------------------------------------------------------------- + +We need to make sure that clusters are able +to talk to each other i.e. specifically the communication between the +Tendermint peers. Set up networking between the clusters using +`Kubernetes Services `_. + +Assuming we have a Planetmint instance ``bdb-instance-1`` residing in Azure data center location ``westeurope`` and we +want to connect to ``bdb-instance-2``, ``bdb-instance-3``, and ``bdb-instance-4`` located in Azure data centers +``eastus``, ``centralus`` and ``westus``, respectively. Unless you already have explicitly set up networking for +``bdb-instance-1`` to communicate with ``bdb-instance-2/3/4`` and +vice versa, we will have to add a Kubernetes Service in each cluster to accomplish this goal in order to set up a +Tendermint P2P network. +It is similar to ensuring that there is a ``CNAME`` record in the DNS +infrastructure to resolve ``bdb-instance-X`` to the host where it is actually available. +We can do this in Kubernetes using a Kubernetes Service of ``type`` +``ExternalName``. + +* This configuration is located in the file ``bigchaindb/bigchaindb-ext-conn-svc.yaml``. + +* Set the name of the ``metadata.name`` to the host name of the Planetmint instance you are trying to connect to. + For instance if you are configuring this service on cluster with ``bdb-instance-1`` then the ``metadata.name`` will + be ``bdb-instance-2`` and vice versa. + +* Set ``spec.ports.port[0]`` to the ``tm-p2p-port`` from the ConfigMap for the other cluster. + +* Set ``spec.ports.port[1]`` to the ``tm-rpc-port`` from the ConfigMap for the other cluster. + +* Set ``spec.externalName`` to the FQDN mapped to NGINX Public IP of the cluster you are trying to connect to. + For more information about the FQDN please refer to: :ref:`Assign DNS name to NGINX Public + IP `. + +.. note:: + This operation needs to be replicated ``n-1`` times per node for a ``n`` node cluster, with the respective FQDNs + we need to communicate with. + + If you are not the system administrator of the cluster, you have to get in + touch with the system administrator/s of the other ``n-1`` clusters and + share with them your instance name (``tendermint-instance-name`` in the ConfigMap) + and the FQDN of the NGINX instance acting as Gateway(set in: :ref:`Assign DNS name to NGINX + Public IP `). + + +.. _verify-and-test-bdb: + +Step 21: Verify the Planetmint Node Setup +----------------------------------------- + +Step 21.1: Testing Internally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To test the setup of your Planetmint node, you could use a Docker container +that provides utilities like ``nslookup``, ``curl`` and ``dig``. +For example, you could use a container based on our +`bigchaindb/toolbox `_ image. +(The corresponding +`Dockerfile `_ +is in the ``bigchaindb/bigchaindb`` repository on GitHub.) +You can use it as below to get started immediately: + +.. code:: bash + + $ kubectl \ + run -it toolbox \ + --image bigchaindb/toolbox \ + --image-pull-policy=Always \ + --restart=Never --rm + +It will drop you to the shell prompt. + +To test the MongoDB instance: + +.. code:: bash + + $ nslookup mdb-instance-0 + + $ dig +noall +answer _mdb-port._tcp.mdb-instance-0.default.svc.cluster.local SRV + + $ curl -X GET http://mdb-instance-0:27017 + +The ``nslookup`` command should output the configured IP address of the service +(in the cluster). +The ``dig`` command should return the configured port numbers. +The ``curl`` command tests the availability of the service. + +To test the Planetmint instance: + +.. code:: bash + + $ nslookup bdb-instance-0 + + $ dig +noall +answer _bdb-api-port._tcp.bdb-instance-0.default.svc.cluster.local SRV + + $ dig +noall +answer _bdb-ws-port._tcp.bdb-instance-0.default.svc.cluster.local SRV + + $ curl -X GET http://bdb-instance-0:9984 + + $ curl -X GET http://bdb-instance-0:9986/pub_key.json + + $ curl -X GET http://bdb-instance-0:26657/abci_info + + $ wsc -er ws://bdb-instance-0:9985/api/v1/streams/valid_transactions + + +To test the OpenResty instance: + +.. code:: bash + + $ nslookup openresty-instance-0 + + $ dig +noall +answer _openresty-svc-port._tcp.openresty-instance-0.default.svc.cluster.local SRV + +To verify if OpenResty instance forwards the requests properly, send a ``POST`` +transaction to OpenResty at post ``80`` and check the response from the backend +Planetmint instance. + + +To test the vanilla NGINX instance: + +.. code:: bash + + $ nslookup ngx-http-instance-0 + + $ dig +noall +answer _public-node-port._tcp.ngx-http-instance-0.default.svc.cluster.local SRV + + $ dig +noall +answer _public-health-check-port._tcp.ngx-http-instance-0.default.svc.cluster.local SRV + + $ wsc -er ws://ngx-http-instance-0/api/v1/streams/valid_transactions + + $ curl -X GET http://ngx-http-instance-0:27017 + +The above curl command should result in the response +``It looks like you are trying to access MongoDB over HTTP on the native driver port.`` + + + +To test the NGINX instance with HTTPS and 3scale integration: + +.. code:: bash + + $ nslookup ngx-instance-0 + + $ dig +noall +answer _public-secure-node-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + $ dig +noall +answer _public-mdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + $ dig +noall +answer _public-insecure-node-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + $ wsc -er wss:///api/v1/streams/valid_transactions + + $ curl -X GET http://:27017 + +The above curl command should result in the response +``It looks like you are trying to access MongoDB over HTTP on the native driver port.`` + + +Step 21.2: Testing Externally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Check the MongoDB monitoring agent on the MongoDB Cloud Manager +portal to verify they are working fine. + +If you are using the NGINX with HTTP support, accessing the URL +``http://:node-frontend-port`` +on your browser should result in a JSON response that shows the Planetmint +server version, among other things. +If you are using the NGINX with HTTPS support, use ``https`` instead of +``http`` above. + +Use the Python Driver to send some transactions to the Planetmint node and +verify that your node or cluster works as expected. + +Next, you can set up log analytics and monitoring, by following our templates: + +* :doc:`../k8s-deployment-template/log-analytics`. diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/revoke-tls-certificate.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/revoke-tls-certificate.rst new file mode 100644 index 0000000..591ea17 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/revoke-tls-certificate.rst @@ -0,0 +1,49 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +How to Revoke an SSL/TLS Certificate +==================================== + +This page enumerates the steps *we* take to revoke a self-signed SSL/TLS +certificate in a Planetmint network. +It can only be done by someone with access to the self-signed CA +associated with the network's managing organization. + +Step 1: Revoke a Certificate +---------------------------- + +Since we used Easy-RSA version 3 to +:ref:`set up the CA `, +we use it to revoke certificates too. + +Go to the following directory (associated with the self-signed CA): +``.../bdb-node-ca/easy-rsa-3.0.1/easyrsa3``. +You need to be aware of the file name used to import the certificate using the +``./easyrsa import-req`` before. Run the following command to revoke a +certificate: + +.. code:: bash + + ./easyrsa revoke + + +This will update the CA database with the revocation details. +The next step is to use the updated database to issue an up-to-date +certificate revocation list (CRL). + +Step 2: Generate a New CRL +-------------------------- + +Generate a new CRL for your infrastructure using: + +.. code:: bash + + ./easyrsa gen-crl + +The generated ``crl.pem`` file needs to be uploaded to your infrastructure to +prevent the revoked certificate from being used again. + +In particlar, the generated ``crl.pem`` file should be sent to all Planetmint node operators in your Planetmint network, so that they can update it in their MongoDB instance and their Planetmint Server instance. diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/server-tls-certificate.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/server-tls-certificate.rst new file mode 100644 index 0000000..060d6aa --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/server-tls-certificate.rst @@ -0,0 +1,102 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _how-to-generate-a-server-certificate-for-mongodb: + +How to Generate a Server Certificate for MongoDB +================================================ + +This page enumerates the steps *we* use to generate a +server certificate for a MongoDB instance. +A server certificate is also referred to as a "member certificate" +in the MongoDB documentation. +We use Easy-RSA. + + +Step 1: Install & Configure Easy–RSA +------------------------------------ + +First create a directory for the server certificate (member cert) and cd into it: + +.. code:: bash + + mkdir member-cert + + cd member-cert + +Then :ref:`install and configure Easy-RSA in that directory `. + + +Step 2: Create the Server Private Key and CSR +--------------------------------------------- + +You can create the server private key and certificate signing request (CSR) +by going into the directory ``member-cert/easy-rsa-3.0.1/easyrsa3`` +and using something like: + +.. note:: + + Please make sure you are fullfilling the requirements for `MongoDB server/member certificates + `_. + +.. code:: bash + + ./easyrsa init-pki + + ./easyrsa --req-cn=mdb-instance-0 --subject-alt-name=DNS:localhost,DNS:mdb-instance-0 gen-req mdb-instance-0 nopass + +You should replace the Common Name (``mdb-instance-0`` above) with the correct name for *your* MongoDB instance in the network, e.g. ``mdb-instance-5`` or ``mdb-instance-12``. (This name is decided by the organization managing the network.) + +You will be prompted to enter the Distinguished Name (DN) information for this certificate. +For each field, you can accept the default value [in brackets] by pressing Enter. + +.. warning:: + + Don't accept the default value of OU (``IT``). Instead, enter the value ``MongoDB-Instance``. + +Aside: You need to provide the ``DNS:localhost`` SAN during certificate generation +for using the ``localhost exception`` in the MongoDB instance. +All certificates can have this attribute without compromising security as the +``localhost exception`` works only the first time. + + +Step 3: Get the Server Certificate Signed +----------------------------------------- + +The CSR file created in the last step +should be located in ``pki/reqs/mdb-instance-0.req`` +(where the integer ``0`` may be different for you). +You need to send it to the organization managing the Planetmint network +so that they can use their CA +to sign the request. +(The managing organization should already have a self-signed CA.) + +If you are the admin of the managing organization's self-signed CA, +then you can import the CSR and use Easy-RSA to sign it. +Go to your ``bdb-node-ca/easy-rsa-3.0.1/easyrsa3/`` +directory and do something like: + +.. code:: bash + + ./easyrsa import-req /path/to/mdb-instance-0.req mdb-instance-0 + + ./easyrsa --subject-alt-name=DNS:localhost,DNS:mdb-instance-0 sign-req server mdb-instance-0 + +Once you have signed it, you can send the signed certificate +and the CA certificate back to the requestor. +The files are ``pki/issued/mdb-instance-0.crt`` and ``pki/ca.crt``. + + +Step 4: Generate the Consolidated Server PEM File +------------------------------------------------- + +MongoDB requires a single, consolidated file containing both the public and +private keys. + +.. code:: bash + + cat /path/to/mdb-instance-0.crt /path/to/mdb-instance-0.key > mdb-instance-0.pem + diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/tectonic-azure.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/tectonic-azure.rst new file mode 100644 index 0000000..eeb05ee --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/tectonic-azure.rst @@ -0,0 +1,149 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Walkthrough: Deploy a Kubernetes Cluster on Azure using Tectonic by CoreOS +========================================================================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +A Planetmint node can be run inside a `Kubernetes `_ +cluster. +This page describes one way to deploy a Kubernetes cluster on Azure using Tectonic. +Tectonic helps in easier cluster management of Kubernetes clusters. + +If you would rather use Azure Container Service to manage Kubernetes Clusters, +please read :doc:`our guide for that `. + + +Step 1: Prerequisites for Deploying Tectonic Cluster +---------------------------------------------------- + +Get an Azure account. Refer to +:ref:`this step in our docs `. + +Create an SSH Key pair for the new Tectonic cluster. Refer to +:ref:`this step in our docs `. + + +Step 2: Get a Tectonic Subscription +----------------------------------- + +CoreOS offers Tectonic for free for up to 10 nodes. + +Sign up for an account `here `__ if you do not +have one already and get a license for 10 nodes. + +Login to your account, go to Overview > Your Account and save the +``CoreOS License`` and the ``Pull Secret`` to your local machine. + + +Step 3: Deploy the cluster on Azure +----------------------------------- + +The latest instructions for deployment can be found +`here `__. + +The following points suggests some customizations for a Planetmint deployment +when following the steps above: + + +#. Set the ``CLUSTER`` variable to the name of the cluster. Also note that the + cluster will be deployed in a resource group named + ``tectonic-cluster-CLUSTER``. + +#. Set the ``tectonic_base_domain`` to ``""`` if you want to use Azure managed + DNS. You will be assigned a ``cloudapp.azure.com`` sub-domain by default and + you can skip the ``Configuring Azure DNS`` section from the Tectonic installation + guide. + +#. Set the ``tectonic_cl_channel`` to ``"stable"`` unless you want to + experiment or test with the latest release. + +#. Set the ``tectonic_cluster_name`` to the ``CLUSTER`` variable defined in + the step above. + +#. Set the ``tectonic_license_path`` and ``tectonic_pull_secret_path`` to the + location where you have stored the ``tectonic-license.txt`` and the + ``config.json`` files downloaded in the previous step. + +#. Set the ``tectonic_etcd_count`` to ``"3"``, so that you have a multi-node + etcd cluster that can tolerate a single node failure. + +#. Set the ``tectonic_etcd_tls_enabled`` to ``"true"`` as this will enable TLS + connectivity between the etcd nodes and their clients. + +#. Set the ``tectonic_master_count`` to ``"3"`` so that you cane tolerate a + single master failure. + +#. Set the ``tectonic_worker_count`` to ``"2"``. + +#. Set the ``tectonic_azure_location`` to ``"westeurope"`` if you want to host + the cluster in Azure's ``westeurope`` datacenter. + +#. Set the ``tectonic_azure_ssh_key`` to the path of the public key created in + the previous step. + +#. We recommend setting up or using a CA(Certificate Authority) to generate Tectonic + Console's server certificate(s) and adding it to your trusted authorities on the client side, + accessing the Tectonic Console i.e. Browser. If you already have a CA(self-signed or otherwise), + Set the ``tectonic_ca_cert`` and ``tectonic_ca_key`` configurations with the content + of PEM-encoded certificate and key files, respectively. For more information about, how to set + up a self-signed CA, Please refer to + :doc:`How to Set up self-signed CA `. + +#. Note that the ``tectonic_azure_client_secret`` is the same as the + ``ARM_CLIENT_SECRET``. + +#. Note that the URL for the Tectonic console using these settings will be the + cluster name set in the configutation file, the datacenter name and + ``cloudapp.azure.com``. For example, if you named your cluster as + ``test-cluster`` and specified the datacenter as ``westeurope``, the Tectonic + console will be available at ``test-cluster.westeurope.cloudapp.azure.com``. + +#. Note that, if you do not specify ``tectonic_ca_cert``, a CA certificate will + be generated automatically and you will encounter the untrusted certificate + message on your client(Browser), when accessing the Tectonic Console. + + +Step 4: Configure kubectl +------------------------- + +#. Refer to `this tutorial + `__ + for instructions on how to download the kubectl configuration files for + your cluster. + +#. Set the ``KUBECONFIG`` environment variable to make ``kubectl`` use the new + config file along with the existing configuration. + +.. code:: bash + + $ export KUBECONFIG=$HOME/.kube/config:/path/to/config/kubectl-config + + # OR to only use the new configuration, try + + $ export KUBECONFIG=/path/to/config/kubectl-config + +Next, you can follow one of our following deployment templates: + +* :doc:`node-on-kubernetes`. + + +Tectonic References +------------------- + +#. https://coreos.com/tectonic/docs/latest/tutorials/azure/install.html +#. https://coreos.com/tectonic/docs/latest/troubleshooting/installer-terraform.html +#. https://coreos.com/tectonic/docs/latest/tutorials/azure/first-app.html \ No newline at end of file diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/template-kubernetes-azure.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/template-kubernetes-azure.rst new file mode 100644 index 0000000..7b90648 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/template-kubernetes-azure.rst @@ -0,0 +1,271 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Template: Deploy a Kubernetes Cluster on Azure +============================================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +A Planetmint node can be run inside a `Kubernetes `_ +cluster. +This page describes one way to deploy a Kubernetes cluster on Azure. + + +.. _get-a-pay-as-you-go-azure-subscription: + +Step 1: Get a Pay-As-You-Go Azure Subscription +---------------------------------------------- + +Microsoft Azure has a Free Trial subscription (at the time of writing), +but it's too limited to run an advanced Planetmint node. +Sign up for a Pay-As-You-Go Azure subscription +via `the Azure website `_. + +You may find that you have to sign up for a Free Trial subscription first. +That's okay: you can have many subscriptions. + + +.. _create-an-ssh-key-pair: + +Step 2: Create an SSH Key Pair +------------------------------ + +You'll want an SSH key pair so you'll be able to SSH +to the virtual machines that you'll deploy in the next step. +(If you already have an SSH key pair, you *could* reuse it, +but it's probably a good idea to make a new SSH key pair +for your Kubernetes VMs and nothing else.) + +See the +:doc:`page about how to generate a key pair for SSH +<../../appendices/generate-key-pair-for-ssh>`. + + +Step 3: Deploy an Azure Container Service (ACS) +----------------------------------------------- + +It's *possible* to deploy an Azure Container Service (ACS) +from the `Azure Portal `_ +(i.e. online in your web browser) +but it's actually easier to do it using the Azure +Command-Line Interface (CLI). + +Microsoft has `instructions to install the Azure CLI 2.0 +on most common operating systems +`_. +Do that. + +If you already *have* the Azure CLI installed, you may want to update it. + +.. warning:: + + ``az component update`` isn't supported if you installed the CLI using some of Microsoft's provided installation instructions. See `the Microsoft docs for update instructions `_. + + +Next, login to your account using: + +.. code:: bash + + $ az login + +It will tell you to open a web page and to copy a code to that page. + +If the login is a success, you will see some information +about all your subscriptions, including the one that is currently +enabled (``"state": "Enabled"``). If the wrong one is enabled, +you can switch to the right one using: + +.. code:: bash + + $ az account set --subscription + +Next, you will have to pick the Azure data center location +where you'd like to deploy your cluster. +You can get a list of all available locations using: + +.. code:: bash + + $ az account list-locations + +Next, create an Azure "resource group" to contain all the +resources (virtual machines, subnets, etc.) associated +with your soon-to-be-deployed cluster. You can name it +whatever you like but avoid fancy characters because they may +confuse some software. + +.. code:: bash + + $ az group create --name --location + + +Example location names are ``koreacentral`` and ``westeurope``. + +Finally, you can deploy an ACS using something like: + +.. code:: bash + + $ az acs create --name \ + --resource-group \ + --master-count 3 \ + --agent-count 3 \ + --admin-username ubuntu \ + --agent-vm-size Standard_L4s \ + --dns-prefix \ + --ssh-key-value ~/.ssh/.pub \ + --orchestrator-type kubernetes \ + --debug --output json + +.. Note:: + The `Azure documentation `_ + has a list of all ``az acs create`` options. + You might prefer a smaller agent VM size, for example. + You can also get a list of the options using: + + .. code:: bash + + $ az acs create --help + + +It takes a few minutes for all the resources to deploy. +You can watch the progress in the `Azure Portal +`_: +go to **Resource groups** (with the blue cube icon) +and click on the one you created +to see all the resources in it. + + +Trouble with the Service Principal? Then Read This! +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the ``az acs create`` command fails with an error message including the text, +"The Service Principal in ServicePrincipalProfile could not be validated", +then we found you can prevent that by creating a Service Principal ahead of time +and telling ``az acs create`` to use that one. (It's supposed to create one, +but sometimes that fails, I guess.) + +Create a new resource group, even if you created one before. They're free anyway: + +.. code:: bash + + $ az login + $ az group create --name \ + --location + +Note the ``id`` in the output. It looks like +``"/subscriptions/369284be-0104-421a-8488-1aeac0caecbb/resourceGroups/examplerg"``. +It can be copied into the next command. +Create a Service Principal using: + +.. code:: bash + + $ az ad sp create-for-rbac --role="Contributor" \ + --scopes= + +Note the ``appId`` and ``password``. +Put those in a new ``az acs create`` command like above, with two new options added: + +.. code:: bash + + $ az acs create ... \ + --service-principal \ + --client-secret + + +.. _ssh-to-your-new-kubernetes-cluster-nodes: + +Optional: SSH to Your New Kubernetes Cluster Nodes +-------------------------------------------------- + +You can SSH to one of the just-deployed Kubernetes "master" nodes +(virtual machines) using: + +.. code:: bash + + $ ssh -i ~/.ssh/ ubuntu@ + +where you can get the IP address or FQDN +of a master node from the Azure Portal. For example: + +.. code:: bash + + $ ssh -i ~/.ssh/mykey123 ubuntu@mydnsprefix.westeurope.cloudapp.azure.com + +.. note:: + + All the master nodes are accessible behind the *same* public IP address and + FQDN. You connect to one of the masters randomly based on the load balancing + policy. + +The "agent" nodes shouldn't get public IP addresses or externally accessible +FQDNs, so you can't SSH to them *directly*, +but you can first SSH to the master +and then SSH to an agent from there using their hostname. +To do that, you could +copy your SSH key pair to the master (a bad idea), +or use SSH agent forwarding (better). +To do the latter, do the following on the machine you used +to SSH to the master: + +.. code:: bash + + $ echo -e "Host \n ForwardAgent yes" >> ~/.ssh/config + +To verify that SSH agent forwarding works properly, +SSH to the one of the master nodes and do: + +.. code:: bash + + $ echo "$SSH_AUTH_SOCK" + +If you get an empty response, +then SSH agent forwarding hasn't been set up correctly. +If you get a non-empty response, +then SSH agent forwarding should work fine +and you can SSH to one of the agent nodes (from a master) +using: + +.. code:: bash + + $ ssh ubuntu@k8s-agent-4AC80E97-0 + +where ``k8s-agent-4AC80E97-0`` is the name +of a Kubernetes agent node in your Kubernetes cluster. +You will have to replace it by the name +of an agent node in your cluster. + + +Optional: Delete the Kubernetes Cluster +--------------------------------------- + +.. code:: bash + + $ az acs delete \ + --name \ + --resource-group + + +Optional: Delete the Resource Group +----------------------------------- + +CAUTION: You might end up deleting resources other than the ACS cluster. + +.. code:: bash + + $ az group delete \ + --name + + +Next, you can :doc:`run a Planetmint node/cluster(BFT) ` +on your new Kubernetes cluster. diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/troubleshoot.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/troubleshoot.rst new file mode 100644 index 0000000..22a7563 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/troubleshoot.rst @@ -0,0 +1,147 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _cluster-troubleshooting: + +Cluster Troubleshooting +======================= + +This page describes some basic issues we have faced while deploying and +operating the cluster. + +1. MongoDB Restarts +------------------- + +We define the following in the ``mongo-ss.yaml`` file: + +.. code:: bash + + resources: + limits: + cpu: 200m + memory: 5G + +When the MongoDB cache occupies a memory greater than 5GB, it is +terminated by the ``kubelet``. +This can usually be verified by logging in to the worker node running MongoDB +container and looking at the syslog (the ``journalctl`` command should usually +work). + +This issue is resolved in +`PR #1757 `_. + +2. 502 Bad Gateway Error on Runscope Tests +------------------------------------------ + +It means that NGINX could not find the appropriate backed to forward the +requests to. This typically happens when: + +#. MongoDB goes down (as described above) and Planetmint, after trying for + ``PLANETMINT_DATABASE_MAXTRIES`` times, gives up. The Kubernetes Planetmint + Deployment then restarts the Planetmint pod. + +#. Planetmint crashes for some reason. We have seen this happen when updating + Planetmint from one version to the next. This usually means the older + connections to the service gets disconnected; retrying the request one more + time, forwards the connection to the new instance and succeed. + + +3. Service Unreachable +---------------------- + +Communication between Kubernetes Services and Deployments fail in +v1.6.6 and before due to a trivial key lookup error for non-existent services +in the ``kubelet``. +This error can be reproduced by restarting any public facing (that is, services +using the cloud load balancer) Kubernetes services, and watching the +``kube-proxy`` failure in its logs. +The solution to this problem is to restart ``kube-proxy`` on the affected +worker/agent node. Login to the worker node and run: + +.. code:: bash + + docker stop `docker ps | grep k8s_kube-proxy | cut -d" " -f1` + + docker logs -f `docker ps | grep k8s_kube-proxy | cut -d" " -f1` + +`This issue `_ is +`fixed in Kubernetes v1.7 `_. + + +4. Single Disk Attached to Multiple Mountpoints in a Container +-------------------------------------------------------------- + +This is currently the issue faced in one of the clusters and being debugged by +the support team at Microsoft. + +The issue was first seen on August 29, 2017 on the Test Network and has been +logged in the `Azure/acs-engine repo on GitHub `_. + +This is apparently fixed in Kubernetes v1.7.2 which include a new disk driver, +but is yet to tested by us. + + +5. MongoDB Monitoring Agent throws a dial error while connecting to MongoDB +--------------------------------------------------------------------------- + +You might see something similar to this in the MongoDB Monitoring Agent logs: + +.. code:: bash + + Failure dialing host without auth. Err: `no reachable servers` + at monitoring-agent/components/dialing.go:278 + at monitoring-agent/components/dialing.go:116 + at monitoring-agent/components/dialing.go:213 + at src/runtime/asm_amd64.s:2086 + + +The first thing to check is if the networking is set up correctly. You can use +the (maybe using the `toolbox` container). + +If everything looks fine, it might be a problem with the ``Preferred +Hostnames`` setting in MongoDB Cloud Manager. If you do need to change the +regular expression, ensure that it is correct and saved properly (maybe try +refreshing the MongoDB Cloud Manager web page to see if the setting sticks). + +Once you update the regular expression, you will need to remove the deployment +and add it again for the Monitoring Agent to discover and connect to the +MongoDB instance correctly. + +More information about this configuration is provided in +:doc:`this document `. + +6. Create a Persistent Volume from existing Azure disk storage Resource +--------------------------------------------------------------------------- +When deleting a k8s cluster, all dynamically-created PVs are deleted, along with the +underlying Azure storage disks (so those can't be used in a new cluster). resources +are also deleted thus cannot be used in a new cluster. This workflow will preserve +the Azure storage disks while deleting the k8s cluster and re-use the same disks on a new +cluster for MongoDB persistent storage without losing any data. + +The template to create two PVs for MongoDB Stateful Set (One for MongoDB data store and +the other for MongoDB config store) is located at ``mongodb/mongo-pv.yaml``. + +You need to configure ``diskName`` and ``diskURI`` in ``mongodb/mongo-pv.yaml`` file. You can get +these values by logging into your Azure portal and going to ``Resource Groups`` and click on your +relevant resource group. From the list of resources click on the storage account resource and +click the container (usually named as ``vhds``) that contains storage disk blobs that are available +for PVs. Click on the storage disk file that you wish to use for your PV and you will be able to +see ``NAME`` and ``URL`` parameters which you can use for ``diskName`` and ``diskURI`` values in +your template respectively and run the following command to create PVs: + +.. code:: bash + + $ kubectl --context apply -f mongodb/mongo-pv.yaml + +.. note:: + + Please make sure the storage disks you are using are not already being used by any + other PVs. To check the existing PVs in your cluster, run the following command + to get PVs and Storage disk file mapping. + + .. code:: bash + + $ kubectl --context get pv --output yaml diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/upgrade-on-kubernetes.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/upgrade-on-kubernetes.rst new file mode 100644 index 0000000..d97ed04 --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/upgrade-on-kubernetes.rst @@ -0,0 +1,122 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Kubernetes Template: Upgrade all Software in a Planetmint Node +============================================================== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +This page outlines how to upgrade all the software associated +with a Planetmint node running on Kubernetes, +including host operating systems, Docker, Kubernetes, +and Planetmint-related software. + + +Upgrade Host OS, Docker and Kubernetes +-------------------------------------- + +Some Kubernetes installation & management systems +can do full or partial upgrades of host OSes, Docker, +or Kubernetes, e.g. +`Tectonic `_, +`Rancher `_, +and +`Kubo `_. +Consult the documentation for your system. + +**Azure Container Service (ACS).** +On Dec. 15, 2016, a Microsoft employee +`wrote `_: +"In the coming months we [the Azure Kubernetes team] will be building managed updates in the ACS service." +At the time of writing, managed updates were not yet available, +but you should check the latest +`ACS documentation `_ +to see what's available now. +Also at the time of writing, ACS only supported Ubuntu +as the host (master and agent) operating system. +You can upgrade Ubuntu and Docker on Azure +by SSHing into each of the hosts, +as documented on +:ref:`another page `. + +In general, you can SSH to each host in your Kubernetes Cluster +to update the OS and Docker. + +.. note:: + + Once you are in an SSH session with a host, + the ``docker info`` command is a handy way to detemine the + host OS (including version) and the Docker version. + +When you want to upgrade the software on a Kubernetes node, +you should "drain" the node first, +i.e. tell Kubernetes to gracefully terminate all pods +on the node and mark it as unscheduleable +(so no new pods get put on the node during its downtime). + +.. code:: + + kubectl drain $NODENAME + +There are `more details in the Kubernetes docs `_, +including instructions to make the node scheduleable again. + +To manually upgrade the host OS, +see the docs for that OS. + +To manually upgrade Docker, see +`the Docker docs `_. + +To manually upgrade all Kubernetes software in your Kubernetes cluster, see +`the Kubernetes docs `_. + + +Upgrade Planetmint-Related Software +----------------------------------- + +We use Kubernetes "Deployments" for NGINX, Planetmint, +and most other Planetmint-related software. +The only exception is MongoDB; we use a Kubernetes +StatefulSet for that. + +The nice thing about Kubernetes Deployments +is that Kubernetes can manage most of the upgrade process. +A typical upgrade workflow for a single Deployment would be: + +.. code:: + + $ KUBE_EDITOR=nano kubectl edit deployment/ + +The ``kubectl edit`` command +opens the specified editor (nano in the above example), +allowing you to edit the specified Deployment *in the Kubernetes cluster*. +You can change the version tag on the Docker image, for example. +Don't forget to save your edits before exiting the editor. +The Kubernetes docs have more information about +`Deployments `_ (including updating them). + + +The upgrade story for the MongoDB StatefulSet is *different*. +(This is because MongoDB has persistent state, +which is stored in some storage associated with a PersistentVolumeClaim.) +At the time of writing, StatefulSets were still in beta, +and they did not support automated image upgrade (Docker image tag upgrade). +We expect that to change. +Rather than trying to keep these docs up-to-date, +we advise you to check out the current +`Kubernetes docs about updating containers in StatefulSets +`_. + + diff --git a/docs/root/source/installation/network-setup/k8s-deployment-template/workflow.rst b/docs/root/source/installation/network-setup/k8s-deployment-template/workflow.rst new file mode 100644 index 0000000..5c682af --- /dev/null +++ b/docs/root/source/installation/network-setup/k8s-deployment-template/workflow.rst @@ -0,0 +1,162 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +.. _kubernetes-template-overview: + +Overview +======== + +.. note:: + + A highly-available Kubernetes cluster requires at least five virtual machines + (three for the master and two for your app's containers). + Therefore we don't recommend using Kubernetes to run a Planetmint node + if that's the only thing the Kubernetes cluster will be running. + Instead, see our `Node Setup <../../node_setup>`_. + If your organization already *has* a big Kubernetes cluster running many containers, + and your organization has people who know Kubernetes, + then this Kubernetes deployment template might be helpful. + +This page summarizes some steps to go through +to set up a Planetmint network. +You can modify them to suit your needs. + +.. _generate-the-blockchain-id-and-genesis-time: + +Generate All Shared Planetmint Setup Parameters +----------------------------------------------- + +There are some shared Planetmint setup paramters that every node operator +in the consortium shares +because they are properties of the Tendermint network. +They look like this: + +.. code:: + + # Tendermint data + BDB_PERSISTENT_PEERS='bdb-instance-1,bdb-instance-2,bdb-instance-3,bdb-instance-4' + BDB_VALIDATORS='bdb-instance-1,bdb-instance-2,bdb-instance-3,bdb-instance-4' + BDB_VALIDATOR_POWERS='10,10,10,10' + BDB_GENESIS_TIME='0001-01-01T00:00:00Z' + BDB_CHAIN_ID='test-chain-rwcPML' + +Those paramters only have to be generated once, by one member of the consortium. +That person will then share the results (Tendermint setup parameters) +with all the node operators. + +The above example parameters are for a network of 4 initial (seed) nodes. +Note how ``BDB_PERSISTENT_PEERS``, ``BDB_VALIDATORS`` and ``BDB_VALIDATOR_POWERS`` are lists +with 4 items each. +**If your consortium has a different number of initial nodes, +then those lists should have that number or items.** +Use ``10`` for all the power values. + +To generate a ``BDB_GENESIS_TIME`` and a ``BDB_CHAIN_ID``, +you can do this: + +.. code:: + + $ mkdir $(pwd)/tmdata + $ docker run --rm -v $(pwd)/tmdata:/tendermint/config tendermint/tendermint:v0.31.5 init + $ cat $(pwd)/tmdata/genesis.json + +You should see something that looks like: + +.. code:: json + + {"genesis_time": "0001-01-01T00:00:00Z", + "chain_id": "test-chain-bGX7PM", + "validators": [ + {"pub_key": + {"type": "ed25519", + "data": "4669C4B966EB8B99E45E40982B2716A9D3FA53B54C68088DAB2689935D7AF1A9"}, + "power": 10, + "name": ""} + ], + "app_hash": "" + } + +The value with ``"genesis_time"`` is ``BDB_GENESIS_TIME`` and +the value with ``"chain_id"`` is ``BDB_CHAIN_ID``. + +Now you have all the Planetmint setup parameters and can share them +with all of the node operators. (They will put them in their ``vars`` file. +We'll say more about that file below.) + + +.. _things-each-node-operator-must-do: + +Things Each Node Operator Must Do +--------------------------------- + +1. Make up an `FQDN `_ +for your Planetmint node (e.g. ``mynode.mycorp.com``). +This is where external users will access the Planetmint HTTP API, for example. +Make sure you've registered the associated domain name (e.g. ``mycorp.com``). + +Get an SSL certificate for your Planetmint node's FQDN. +Also get the root CA certificate and all intermediate certificates. +They should all be provided by your SSL certificate provider. +Put all those certificates together in one certificate chain file in the following order: + +- Domain certificate (i.e. the one you ordered for your FQDN) +- All intermediate certificates +- Root CA certificate + +DigiCert has `a web page explaining certificate chains `_. + +You will put the path to that certificate chain file in the ``vars`` file, +when you configure your node later. + +2a. If your Planetmint node will use 3scale for API authentication, monitoring and billing, +you will need all relevant 3scale settings and credentials. + +2b. If your Planetmint node will not use 3scale, then write authorization will be granted +to all POST requests with a secret token in the HTTP headers. +(All GET requests are allowed to pass.) +You can make up that ``SECRET_TOKEN`` now. +For example, ``superSECRET_token4-POST*requests``. +You will put it in the ``vars`` file later. +Every Planetmint node in a Planetmint network can have a different secret token. +To make an HTTP POST request to your Planetmint node, +you must include an HTTP header named ``X-Secret-Access-Token`` +and set it equal to your secret token, e.g. + +``X-Secret-Access-Token: superSECRET_token4-POST*requests`` + + +3. Deploy a Kubernetes cluster for your Planetmint node. We have some instructions for how to +:doc:`Deploy a Kubernetes cluster on Azure <../k8s-deployment-template/template-kubernetes-azure>`. + +.. warning:: + + In theory, you can deploy your Planetmint node to any Kubernetes cluster, but there can be differences + between different Kubernetes clusters, especially if they are running different versions of Kubernetes. + We tested this Kubernetes Deployment Template on Azure ACS in February 2018 and at that time + ACS was deploying a **Kubernetes 1.7.7** cluster. If you can force your cluster to have that version of Kubernetes, + then you'll increase the likelihood that everything will work. + +4. Deploy your Planetmint node inside your new Kubernetes cluster. +You will fill up the ``vars`` file, +then you will run a script which reads that file to generate some Kubernetes config files, +you will send those config files to your Kubernetes cluster, +and then you will deploy all the stuff that you need to have a Planetmint node. + +⟶ Proceed to :ref:`deploy your Planetmint node `. + +.. raw:: html + +
+
+
+
+
+
+
+
+
+
+
diff --git a/docs/root/source/installation/network-setup/network-setup.md b/docs/root/source/installation/network-setup/network-setup.md new file mode 100644 index 0000000..74bdf37 --- /dev/null +++ b/docs/root/source/installation/network-setup/network-setup.md @@ -0,0 +1,208 @@ + + +# How to Set Up a Planetmint Network + +You can setup or connect to a network once you have a single node running. +Until now, everything could be done by a node operator, by themselves. +Now the node operators, also called **Members**, must share some information +with each other, so they can form a network. + +There is one special Member who helps coordinate everyone: the **Coordinator**. + +## Member: Share hostname, pub_key.value and node_id + +Each Planetmint node is identified by its: + +* `hostname`, i.e. the node's DNS subdomain, such as `bnode.example.com`, or its IP address, such as `46.145.17.32` +* Tendermint `pub_key.value` +* Tendermint `node_id` + +The Tendermint `pub_key.value` is stored +in the file `$HOME/.tendermint/config/priv_validator.json`. +That file should look like: + +```json +{ + "address": "E22D4340E5A92E4A9AD7C62DA62888929B3921E9", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "P+aweH73Hii8RyCmNWbwPsa9o4inq3I+0fSfprVkZa0=" + }, + "last_height": "0", + "last_round": "0", + "last_step": 0, + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "AHBiZXdZhkVZoPUAiMzClxhl0VvUp7Xl3YT6GvCc93A/5rB4fvceKLxHIKY1ZvA+xr2jiKercj7R9J+mtWRlrQ==" + } +} +``` + +To get your Tendermint `node_id`, run the command: + +``` +tendermint show_node_id +``` + +An example `node_id` is `9b989cd5ac65fec52652a457aed6f5fd200edc22`. + +**Share your `hostname`, `pub_key.value` and `node_id` with all other Members.** + +## Coordinator: Create & Share the genesis.json File + +At this point the Coordinator should have received the data +from all the Members, and should combine them in the file +`$HOME/.tendermint/config/genesis.json`: + +```json +{ + "genesis_time":"0001-01-01T00:00:00Z", + "chain_id":"test-chain-la6HSr", + "consensus_params":{ + "block_size_params":{ + "max_bytes":"22020096", + "max_txs":"10000", + "max_gas":"-1" + }, + "tx_size_params":{ + "max_bytes":"10240", + "max_gas":"-1" + }, + "block_gossip_params":{ + "block_part_size_bytes":"65536" + }, + "evidence_params":{ + "max_age":"100000" + } + }, + "validators":[ + { + "pub_key":{ + "type":"tendermint/PubKeyEd25519", + "value":"" + }, + "power":10, + "name":"" + }, + { + "pub_key":{ + "type":"tendermint/PubKeyEd25519", + "value":"" + }, + "power":10, + "name":"" + }, + { + "...":{ + + }, + + }, + { + "pub_key":{ + "type":"tendermint/PubKeyEd25519", + "value":"" + }, + "power":10, + "name":"" + } + ], + "app_hash":"" +} +``` + +**Note:** The above `consensus_params` in the `genesis.json` +are default values. + +The new `genesis.json` file contains the data that describes the Network. +The key `name` is the Member's moniker; it can be any valid string, +but put something human-readable like `"Alice's Node Shop"`. + +At this point, the Coordinator must share the new `genesis.json` file with all Members. + +## Member: Connect to the Other Members + +At this point the Member should have received the `genesis.json` file. + +The Member must copy the `genesis.json` file +into their local `$HOME/.tendermint/config` directory. +Every Member now shares the same `chain_id` and `genesis_time` (used to identify the Network), +and the same list of `validators`. + +Each Member must edit their `$HOME/.tendermint/config/config.toml` file +and make the following changes: + +``` +moniker = "Name of our node" +create_empty_blocks = false +log_level = "main:info,state:info,*:error" + +persistent_peers = "@:26656,\ +@:26656,\ +@:26656," + +send_rate = 102400000 +recv_rate = 102400000 + +recheck = false +``` + +Note: The list of `persistent_peers` doesn't have to include all nodes +in the network. + +## Member: Start MongoDB + +If you installed MongoDB using `sudo apt install mongodb`, then MongoDB should already be running in the background. You can check using `systemctl status mongodb`. + +If MongoDB isn't running, then you can start it using the command `mongod`, but that will run it in the foreground. If you want to run it in the background (so it will continue running after you logout), you can use `mongod --fork --logpath /var/log/mongodb.log`. (You might have to create the `/var/log` directory if it doesn't already exist.) + +If you installed MongoDB using `sudo apt install mongodb`, then a MongoDB startup script should already be installed (so MongoDB will start automatically when the machine is restarted). Otherwise, you should install a startup script for MongoDB. + +## Member: Start Planetmint and Tendermint Using Monit + +This section describes how to manage the Planetmint and Tendermint processes using [Monit][monit], a small open-source utility for managing and monitoring Unix processes. Planetmint and Tendermint are managed together, because if Planetmint is stopped (or crashes) and is restarted, *Tendermint won't try reconnecting to it*. (That's not a bug. It's just how Tendermint works.) + +Install Monit: + +``` +sudo apt install monit +``` + +If you installed the `bigchaindb` Python package as above, you should have the `bigchaindb-monit-config` script in your `PATH` now. Run the script to build a configuration file for Monit: + +``` +bigchaindb-monit-config +``` + +Run Monit as a daemon, instructing it to wake up every second to check on processes: + +``` +monit -d 1 +``` + +Monit will run the Planetmint and Tendermint processes and restart them when they crash. If the root `bigchaindb_` process crashes, Monit will also restart the Tendermint process. + +You can check the status by running `monit status` or `monit summary`. + +By default, it will collect program logs into the `~/.bigchaindb-monit/logs` folder. + +To learn more about Monit, use `monit -h` (help) or read [the Monit documentation][monit-manual]. + +Check `bigchaindb-monit-config -h` if you want to arrange a different folder for logs or some of the Monit internal artifacts. + +If you want to start and manage the Planetmint and Tendermint processes yourself, then look inside the file [bigchaindb/pkg/scripts/bigchaindb-monit-config](https://github.com/bigchaindb/bigchaindb/blob/master/pkg/scripts/bigchaindb-monit-config) to see how *it* starts Planetmint and Tendermint. + +## How Others Can Access Your Node + +If you followed the above instructions, then your node should be publicly-accessible with Planetmint Root URL `https://hostname` or `http://hostname:9984`. That is, anyone can interact with your node using the [Planetmint HTTP API](../api/http-client-server-api) exposed at that address. The most common way to do that is to use one of the [Planetmint Drivers](../../drivers/index). + +[bdb:software]: https://github.com/bigchaindb/bigchaindb/ +[bdb:pypi]: https://pypi.org/project/Planetmint/#history +[tendermint:releases]: https://github.com/tendermint/tendermint/releases +[monit]: https://www.mmonit.com/monit +[monit-manual]: https://mmonit.com/monit/documentation/monit.html diff --git a/docs/root/source/installation/network-setup/networks.md b/docs/root/source/installation/network-setup/networks.md new file mode 100644 index 0000000..9943c04 --- /dev/null +++ b/docs/root/source/installation/network-setup/networks.md @@ -0,0 +1,44 @@ + + +# Planetmint Networks + +A **Planetmint network** is a set of connected **Planetmint nodes**, managed by a **Planetmint consortium** (i.e. an organization). Those terms are defined in the [Planetmint Terminology page](https://docs.bigchaindb.com/en/latest/terminology.html). + +## Consortium Structure & Governance + +The consortium might be a company, a foundation, a cooperative, or [some other form of organization](https://en.wikipedia.org/wiki/Organizational_structure). +It must make many decisions, e.g. How will new members be added? Who can read the stored data? What kind of data will be stored? +A governance process is required to make those decisions, and therefore one of the first steps for any new consortium is to specify its governance process (if one doesn't already exist). +This documentation doesn't explain how to create a consortium, nor does it outline the possible governance processes. + +It's worth noting that the decentralization of a Planetmint network depends, +to some extent, on the decentralization of the associated consortium. See the pages about [decentralization](https://docs.bigchaindb.com/en/latest/decentralized.html) and [node diversity](https://docs.bigchaindb.com/en/latest/diversity.html). + +## DNS Records and SSL Certificates + +We now describe how *we* set up the external (public-facing) DNS records for a Planetmint network. Your consortium may opt to do it differently. +There were several goals: + +* Allow external users/clients to connect directly to any Planetmint node in the network (over the internet), if they want. +* Each Planetmint node operator should get an SSL certificate for their Planetmint node, so that their Planetmint node can serve the [Planetmint HTTP API](../api/http-client-server-api) via HTTPS. (The same certificate might also be used to serve the [WebSocket API](../api/websocket-event-stream-api).) +* There should be no sharing of SSL certificates among Planetmint node operators. +* Optional: Allow clients to connect to a "random" Planetmint node in the network at one particular domain (or subdomain). + +### Node Operator Responsibilities + +1. Register a domain (or use one that you already have) for your Planetmint node. You can use a subdomain if you like. For example, you might opt to use `abc-org73.net`, `api.dynabob8.io` or `figmentdb3.ninja`. +2. Get an SSL certificate for your domain or subdomain, and properly install it in your node (e.g. in your NGINX instance). +3. Create a DNS A Record mapping your domain or subdomain to the public IP address of your node (i.e. the one that serves the Planetmint HTTP API). + +### Consortium Responsibilities + +Optional: The consortium managing the Planetmint network could register a domain name and set up CNAME records mapping that domain name (or one of its subdomains) to each of the nodes in the network. For example, if the consortium registered `bdbnetwork.io`, they could set up CNAME records like the following: + +* CNAME record mapping `api.bdbnetwork.io` to `abc-org73.net` +* CNAME record mapping `api.bdbnetwork.io` to `api.dynabob8.io` +* CNAME record mapping `api.bdbnetwork.io` to `figmentdb3.ninja` diff --git a/docs/root/source/installation/node-setup/all-in-one-bigchaindb.md b/docs/root/source/installation/node-setup/all-in-one-bigchaindb.md new file mode 100644 index 0000000..d3806a4 --- /dev/null +++ b/docs/root/source/installation/node-setup/all-in-one-bigchaindb.md @@ -0,0 +1,92 @@ + + +# Run Planetmint with all-in-one Docker + +For those who like using Docker and wish to experiment with Planetmint in +non-production environments, we currently maintain a Planetmint all-in-one +Docker image and a +`Dockerfile-all-in-one` that can be used to build an image for `bigchaindb`. + +This image contains all the services required for a Planetmint node i.e. + +- Planetmint Server +- MongoDB +- Tendermint + +**Note:** **NOT for Production Use:** *This is an single node opinionated image not well suited for a network deployment.* +*This image is to help quick deployment for early adopters, for a more standard approach please refer to one of our deployment guides:* + +- [Planetmint developer setup guides](https://docs.bigchaindb.com/projects/contributing/en/latest/dev-setup-coding-and-contribution-process/index.html). +- [Planetmint with Kubernetes](http://docs.bigchaindb.com/projects/server/en/latest/k8s-deployment-template/index.html). + +## Prerequisite(s) +- [Docker](https://docs.docker.com/engine/installation/) + +## Pull and Run the Image from Docker Hub + +With Docker installed, you can proceed as follows. + +In a terminal shell, pull the latest version of the Planetmint all-in-one Docker image using: +```text +$ docker pull bigchaindb/bigchaindb:all-in-one + +$ docker run \ + --detach \ + --name bigchaindb \ + --publish 9984:9984 \ + --publish 9985:9985 \ + --publish 27017:27017 \ + --publish 26657:26657 \ + --volume $HOME/bigchaindb_docker/mongodb/data/db:/data/db \ + --volume $HOME/bigchaindb_docker/mongodb/data/configdb:/data/configdb \ + --volume $HOME/bigchaindb_docker/tendermint:/tendermint \ + bigchaindb/bigchaindb:all-in-one +``` + +Let's analyze that command: + +* `docker run` tells Docker to run some image +* `--detach` run the container in the background +* `publish 9984:9984` map the host port `9984` to the container port `9984` + (the Planetmint API server) + * `9985` Planetmint Websocket server + * `27017` Default port for MongoDB + * `26657` Tendermint RPC server +* `--volume "$HOME/bigchaindb_docker/mongodb:/data"` map the host directory + `$HOME/bigchaindb_docker/mongodb` to the container directory `/data`; + this allows us to have the data persisted on the host machine, + you can read more in the [official Docker + documentation](https://docs.docker.com/engine/tutorials/dockervolumes) + * `$HOME/bigchaindb_docker/tendermint:/tendermint` to persist Tendermint data. +* `bigchaindb/bigchaindb:all-in-one` the image to use. All the options after the container name are passed on to the entrypoint inside the container. + +## Verify + +```text +$ docker ps | grep bigchaindb +``` + +Send your first transaction using [Planetmint drivers](../../drivers/index). + + +## Building Your Own Image + +Assuming you have Docker installed, you would proceed as follows. + +In a terminal shell: +```text +git clone git@github.com:bigchaindb/bigchaindb.git +cd bigchaindb/ +``` + +Build the Docker image: +```text +docker build --file Dockerfile-all-in-one --tag . +``` + +Now you can use your own image to run Planetmint all-in-one container. diff --git a/docs/root/source/installation/node-setup/aws-setup.md b/docs/root/source/installation/node-setup/aws-setup.md new file mode 100644 index 0000000..7ec8604 --- /dev/null +++ b/docs/root/source/installation/node-setup/aws-setup.md @@ -0,0 +1,65 @@ + + +# Basic AWS Setup + +Before you can deploy anything on AWS, you must do a few things. + +## Get an AWS Account + +If you don't already have an AWS account, you can [sign up for one for free at aws.amazon.com](https://aws.amazon.com/). + +## Install the AWS Command-Line Interface + +To install the AWS Command-Line Interface (CLI), just do: + +```text +pip install awscli +``` + +## Create an AWS Access Key + +The next thing you'll need is AWS access keys (access key ID and secret access key). If you don't have those, see [the AWS documentation about access keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys). + +You should also pick a default AWS region name (e.g. `eu-central-1`). The AWS documentation has [a list of them](http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region). + +Once you've got your AWS access key, and you've picked a default AWS region name, go to a terminal session and enter: + +```text +aws configure +``` + +and answer the four questions. For example: + +```text +AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE +AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Default region name [None]: eu-central-1 +Default output format [None]: [Press Enter] +``` + +This writes two files: `~/.aws/credentials` and `~/.aws/config`. AWS tools and packages look for those files. + +## Generate an RSA Key Pair for SSH + +Eventually, you'll have one or more instances (virtual machines) running on AWS and you'll want to SSH to them. To do that, you need a public/private key pair. The public key will be sent to AWS, and you can tell AWS to put it in any instances you provision there. You'll keep the private key on your local workstation. + +See the appendix [page about how to generate a key pair for SSH](../appendices/generate-key-pair-for-ssh). + +## Send the Public Key to AWS + +To send the public key to AWS, use the AWS Command-Line Interface: + +```text +aws ec2 import-key-pair \ +--key-name "" \ +--public-key-material file://~/.ssh/.pub +``` + +If you're curious why there's a `file://` in front of the path to the public key, see issue [aws/aws-cli#41 on GitHub](https://github.com/aws/aws-cli/issues/41). + +If you want to verify that your key pair was imported by AWS, go to [the Amazon EC2 console](https://console.aws.amazon.com/ec2/v2/home), select the region you gave above when you did `aws configure` (e.g. eu-central-1), click on **Key Pairs** in the left sidebar, and check that `` is listed. diff --git a/docs/root/source/installation/node-setup/bigchaindb-cli.md b/docs/root/source/installation/node-setup/bigchaindb-cli.md new file mode 100644 index 0000000..44865f8 --- /dev/null +++ b/docs/root/source/installation/node-setup/bigchaindb-cli.md @@ -0,0 +1,206 @@ + + +# Command Line Interface (CLI) + +The command-line command to interact with Planetmint Server is `bigchaindb`. + + +## bigchaindb \-\-help + +Show help for the `bigchaindb` command. `bigchaindb -h` does the same thing. + + +## bigchaindb \-\-version + +Show the version number. `bigchaindb -v` does the same thing. + + +## bigchaindb configure + +Generate a local configuration file (which can be used to set some or all [Planetmint node configuration settings](configuration)). It will ask you for the values of some configuration settings. +If you press Enter for a value, it will use the default value. + +At this point, only one database backend is supported: `localmongodb`. + +If you use the `-c` command-line option, it will generate the file at the specified path: +```text +bigchaindb -c path/to/new_config.json configure localmongodb +``` + +If you don't use the `-c` command-line option, the file will be written to `$HOME/.bigchaindb` (the default location where Planetmint looks for a config file, if one isn't specified). + +If you use the `-y` command-line option, then there won't be any interactive prompts: it will use the default values for all the configuration settings. +```text +bigchaindb -y configure localmongodb +``` + + +## bigchaindb show-config + +Show the values of the [Planetmint node configuration settings](configuration). + + +## bigchaindb init + +Create a backend database (local MongoDB), all database tables/collections, +various backend database indexes, and the genesis block. + + +## bigchaindb drop + +Drop (erase) the backend database (the local MongoDB database used by this node). +You will be prompted to make sure. +If you want to force-drop the database (i.e. skipping the yes/no prompt), then use `bigchaindb -y drop` + + +## bigchaindb start + +Start Planetmint. It always begins by trying a `bigchaindb init` first. See the documentation for `bigchaindb init`. +The database initialization step is optional and can be skipped by passing the `--no-init` flag, i.e. `bigchaindb start --no-init`. + +### Options + +The log level for the console can be set via the option `--log-level` or its +abbreviation `-l`. Example: + +```bash +$ bigchaindb --log-level INFO start +``` + +The allowed levels are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`. +For an explanation regarding these levels please consult the +[Logging Levels](https://docs.python.org/3.6/library/logging.html#levels) +section of Python's documentation. + +For a more fine-grained control over the logging configuration you can use the +configuration file as documented under +[Configuration Settings](configuration). + + +## bigchaindb election + +Manage elections to govern the Planetmint network. The specifics of the election process are defined in [BEP-18](https://github.com/bigchaindb/BEPs/tree/master/18). + +Election management is broken into several subcommands. Below is the command line syntax for each of them. + +### election new + +Create a new election which proposes a change to the Planetmint network. + +If the command succeeds, it will post an election transaction and output `election_id`. + +The election proposal consists of vote tokens allocated to every current validator proportional to his voting power. Validators spend their votes to approve the election using the [election-approve command](#election-approve). + +Every election has a type. Currently supported types are `upsert-validator` and `chain-migration`. Their transaction operations are `VALIDATOR_ELECTION` and `CHAIN_MIGRATION` accordingly. See below for how to create an election of a particular type. + +Note that elections can only be proposed and approved by existing validators. + +#### election new upsert-validator + +Create an election to add, update, or remove a validator. + + +```bash +$ bigchaindb election new upsert-validator --private-key +``` + +- `` is the public key of the node to be added/updated/removed. The encoding and type of the key have to match those specified in `genesis.json` in the supported Tendermint version. +- `` is the new power for the validator. To remove the validator, set the power to `0`. +- `` is the node identifier from Tendermint. A node operator can learn his node identifier by executing `tendermint show_node_id`. +- `` is the path to the private key of the validator who proposes the election. Tendermint places it at `.tendermint/config/priv_validator.json`. + +Example: + +```bash +$ bigchaindb election new upsert-validator HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52GnGuPY= 1 fb7140f03a4ffad899fabbbf655b97e0321add66 --private-key /home/user/.tendermint/config/priv_validator.json +[SUCCESS] Submitted proposal with id: 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa +``` + +A successful execution of the above command does not imply the validator set has been updated but rather the proposal has been accepted by the network. +Once `election_id` has been generated, the proposer should share it with other validators of the network (e.g. via email) and ask them to approve the proposal. + +Note that election proposers do not automatically approve elections by proposing them. + +For more details about how validator set changes work, refer to [BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21). + +#### election new chain-migration + +Create an election to halt block production, to coordinate on making a Tendermint upgrade with a backwards-incompatible chain. + + +```bash +$ bigchaindb election new chain-migration --private-key +``` + +- `` is the path to the private key of the validator who proposes the election. Tendermint places it at `.tendermint/config/priv_validator.json`. + + +Example: + +```bash +$ bigchaindb election new migration --private-key /home/user/.tendermint/config/priv_validator.json +[SUCCESS] Submitted proposal with id: 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa +``` + +Concluded chain migration elections halt block production at whichever block height they are approved. +Afterwards, validators are supposed to upgrade Tendermint, set new `chain_id`, `app_hash`, and `validators` (to learn these values, use the [election show](#election-show) command) in `genesis.json`, make and save a MongoDB dump, and restart the system. + + +For more details about how chain migrations work, refer to [Type 3 scenarios in BEP-42](https://github.com/bigchaindb/BEPs/tree/master/42). + +### election approve + +Approve an election by voting for it. The command places a `VOTE` transaction, spending all of the validator's vote tokens to the election address. + + + ```bash +$ bigchaindb election approve --private-key +``` + +- `election-id` is the election identifier the approval is given for. +- `` is the path to the private key of the validator who votes for the election. Tendermint places it at `.tendermint/config/priv_validator.json`. + +Example: + ```bash +$ bigchaindb election approve 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa --private-key /home/user/.tendermint/config/priv_validator.json +[SUCCESS] Your vote has been submitted +``` + +Once a proposal has been approved by the sufficient amount of validators (contributing more than `2/3` of the total voting power), the proposed change is applied to the network. + +### election show + +Retrieves the information about elections. + + +```bash +$ bigchaindb election show + +status= +``` + +`status` has three possible values: + +- `ongoing`, if the election can be concluded but has not yet collected enough votes, +- `concluded`, if the election has been concluded, +- `inconclusive`, if the validator set changed while the election was in process, rendering it undecidable. + +After a chain migration is concluded, the `show` command also outputs `chain_id`, `app_hash`, and `validators` for `genesis.json` of the new chain. + +## bigchaindb tendermint-version + +Show the Tendermint versions supported by Planetmint server. +```bash +$ bigchaindb tendermint-version +{ + "description": "Planetmint supports the following Tendermint version(s)", + "tendermint": [ + "0.22.8" + ] +} +``` diff --git a/docs/root/source/installation/node-setup/bigchaindb-node-ansible.md b/docs/root/source/installation/node-setup/bigchaindb-node-ansible.md new file mode 100644 index 0000000..53733d9 --- /dev/null +++ b/docs/root/source/installation/node-setup/bigchaindb-node-ansible.md @@ -0,0 +1,7 @@ +# Setting up a network of nodes with the Ansible script + +You can find one of the installation methods with Ansible on GitHub at: + +[Ansible script](https://github.com/bigchaindb/bigchaindb-node-ansible) + +It allows to install Planetmint, MongoDB, Tendermint, and python, and then connect nodes into a network. Current tested machine is Ubuntu 18.04. \ No newline at end of file diff --git a/docs/root/source/installation/node-setup/configuration.md b/docs/root/source/installation/node-setup/configuration.md new file mode 100644 index 0000000..330d359 --- /dev/null +++ b/docs/root/source/installation/node-setup/configuration.md @@ -0,0 +1,398 @@ + + +# Configuration Settings + +Every Planetmint Server configuration setting has two names: a config-file name and an environment variable name. For example, one of the settings has the config-file name `database.host` and the environment variable name `PLANETMINT_DATABASE_HOST`. Here are some more examples: + +`database.port` ↔ `PLANETMINT_DATABASE_PORT` + +`database.keyfile_passphrase` ↔ `PLANETMINT_DATABASE_KEYFILE_PASSPHRASE` + +`server.bind` ↔ `PLANETMINT_SERVER_BIND` + +The value of each setting is determined according to the following rules: + +* If it's set by an environment variable, then use that value +* Otherwise, if it's set in a local config file, then use that value +* Otherwise, use the default value + +The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell Planetmint to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` +or using the `PLANETMINT_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. +Note that the `-c` command line option will always take precedence if both the `PLANETMINT_CONFIG_PATH` and the `-c` command line option are used. + +You can read the current default values in the file [bigchaindb/\_\_init\_\_.py](https://github.com/bigchaindb/bigchaindb/blob/master/bigchaindb/__init__.py). (The link is to the latest version.) + +Running `bigchaindb -y configure localmongodb` will generate a local config file in `$HOME/.bigchaindb` with all the default values. + +## database.* + +The settings with names of the form `database.*` are for the backend database +(currently only MongoDB). They are: + +* `database.backend` can only be `localmongodb`, currently. +* `database.host` is the hostname (FQDN) of the backend database. +* `database.port` is self-explanatory. +* `database.name` is a user-chosen name for the database inside MongoDB, e.g. `bigchain`. +* `database.connection_timeout` is the maximum number of milliseconds that Planetmint will wait before giving up on one attempt to connect to the backend database. +* `database.max_tries` is the maximum number of times that Planetmint will try to establish a connection with the backend database. If 0, then it will try forever. +* `database.replicaset` is the name of the MongoDB replica set. The default value is `null` because in Planetmint 2.0+, each Planetmint node has its own independent MongoDB database and no replica set is necessary. Replica set must already exist if this option is configured, Planetmint will not create it. +* `database.ssl` must be `true` or `false`. It tells Planetmint Server whether it should connect to MongoDB using TLS/SSL or not. The default value is `false`. + +There are three ways for Planetmint Server to authenticate itself with MongoDB (or a specific MongoDB database): no authentication, username/password, and x.509 certificate authentication. + +**No Authentication** + +If you use all the default Planetmint configuration settings, then no authentication will be used. + +**Username/Password Authentication** + +To use username/password authentication, a MongoDB instance must already be running somewhere (maybe in another machine), it must already have a database for use by Planetmint (usually named `bigchain`, which is the default `database.name`), and that database must already have a "readWrite" user with associated username and password. To create such a user, login to your MongoDB instance as Admin and run the following commands: + +```text +use +db.createUser({user: "", pwd: "", roles: [{role: "readWrite", db: ""}]}) +``` + +* `database.login` is the user's username. +* `database.password` is the user's password, given in plaintext. +* `database.ca_cert`, `database.certfile`, `database.keyfile`, `database.crlfile`, and `database.keyfile_passphrase` are not used so they can have their default values. + +**x.509 Certificate Authentication** + +To use x.509 certificate authentication, a MongoDB instance must be running somewhere (maybe in another machine), it must already have a database for use by Planetmint (usually named `bigchain`, which is the default `database.name`), and that database must be set up to use x.509 authentication. See the MongoDB docs about how to do that. + +* `database.login` is the user's username. +* `database.password` isn't used so the default value (`null`) is fine. +* `database.ca_cert`, `database.certfile`, `database.keyfile` and `database.crlfile` are the paths to the CA, signed certificate, private key and certificate revocation list files respectively. +* `database.keyfile_passphrase` is the private key decryption passphrase, specified in plaintext. + +**Example using environment variables** + +```text +export PLANETMINT_DATABASE_BACKEND=localmongodb +export PLANETMINT_DATABASE_HOST=localhost +export PLANETMINT_DATABASE_PORT=27017 +export PLANETMINT_DATABASE_NAME=database8 +export PLANETMINT_DATABASE_CONNECTION_TIMEOUT=5000 +export PLANETMINT_DATABASE_MAX_TRIES=3 +``` + +**Default values** + +If (no environment variables were set and there's no local config file), or you used `bigchaindb -y configure localmongodb` to create a default local config file for a `localmongodb` backend, then the defaults will be: + +```js +"database": { + "backend": "localmongodb", + "host": "localhost", + "port": 27017, + "name": "bigchain", + "connection_timeout": 5000, + "max_tries": 3, + "replicaset": null, + "login": null, + "password": null + "ssl": false, + "ca_cert": null, + "certfile": null, + "keyfile": null, + "crlfile": null, + "keyfile_passphrase": null, +} +``` + +## server.* + +`server.bind`, `server.loglevel` and `server.workers` +are settings for the [Gunicorn HTTP server](http://gunicorn.org/), which is used to serve the [HTTP client-server API](../api/http-client-server-api). + +`server.bind` is where to bind the Gunicorn HTTP server socket. It's a string. It can be any valid value for [Gunicorn's bind setting](http://docs.gunicorn.org/en/stable/settings.html#bind). For example: + +* If you want to allow IPv4 connections from anyone, on port 9984, use `0.0.0.0:9984` +* If you want to allow IPv6 connections from anyone, on port 9984, use `[::]:9984` + +In a production setting, we recommend you use Gunicorn behind a reverse proxy server such as NGINX. If Gunicorn and the reverse proxy are running on the same machine, then you can use `localhost:9984` (the default value), meaning Gunicorn will talk to the reverse proxy on port 9984. The reverse proxy could then be bound to port 80 (for HTTP) or port 443 (for HTTPS), so that external clients would connect using that port. For example: + +[External clients]---(port 443)---[NGINX]---(port 9984)---[Gunicorn / Planetmint Server] + +If Gunicorn and the reverse proxy are running on different machines, then `server.bind` should be `hostname:9984`, where hostname is the IP address or [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) of the reverse proxy. + +There's [more information about deploying behind a reverse proxy in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) + +`server.loglevel` sets the log level of Gunicorn's Error log outputs. See +[Gunicorn's documentation](http://docs.gunicorn.org/en/latest/settings.html#loglevel) +for more information. + +`server.workers` is [the number of worker processes](http://docs.gunicorn.org/en/stable/settings.html#workers) for handling requests. If set to `None`, the value will be (2 × cpu_count + 1). Each worker process has a single thread. The HTTP server will be able to handle `server.workers` requests simultaneously. + +**Example using environment variables** + +```text +export PLANETMINT_SERVER_BIND=0.0.0.0:9984 +export PLANETMINT_SERVER_LOGLEVEL=debug +export PLANETMINT_SERVER_WORKERS=5 +``` + +**Example config file snippet** + +```js +"server": { + "bind": "0.0.0.0:9984", + "loglevel": "debug", + "workers": 5, +} +``` + +**Default values (from a config file)** + +```js +"server": { + "bind": "localhost:9984", + "loglevel": "info", + "workers": null, +} +``` + +## wsserver.* + + +### wsserver.scheme, wsserver.host and wsserver.port + +These settings are for the +[aiohttp server](https://aiohttp.readthedocs.io/en/stable/index.html), +which is used to serve the +[WebSocket Event Stream API](../api/websocket-event-stream-api). +`wsserver.scheme` should be either `"ws"` or `"wss"` +(but setting it to `"wss"` does *not* enable SSL/TLS). +`wsserver.host` is where to bind the aiohttp server socket and +`wsserver.port` is the corresponding port. +If you want to allow connections from anyone, on port 9985, +set `wsserver.host` to 0.0.0.0 and `wsserver.port` to 9985. + +**Example using environment variables** + +```text +export PLANETMINT_WSSERVER_SCHEME=ws +export PLANETMINT_WSSERVER_HOST=0.0.0.0 +export PLANETMINT_WSSERVER_PORT=9985 +``` + +**Example config file snippet** + +```js +"wsserver": { + "scheme": "wss", + "host": "0.0.0.0", + "port": 65000 +} +``` + +**Default values (from a config file)** + +```js +"wsserver": { + "scheme": "ws", + "host": "localhost", + "port": 9985 +} +``` + +### wsserver.advertised_scheme, wsserver.advertised_host and wsserver.advertised_port + +These settings are for the advertising the Websocket URL to external clients in +the root API endpoint. These configurations might be useful if your deployment +is hosted behind a firewall, NAT, etc. where the exposed public IP or domain is +different from where Planetmint is running. + +**Example using environment variables** + +```text +export PLANETMINT_WSSERVER_ADVERTISED_SCHEME=wss +export PLANETMINT_WSSERVER_ADVERTISED_HOST=mybigchaindb.com +export PLANETMINT_WSSERVER_ADVERTISED_PORT=443 +``` + +**Example config file snippet** + +```js +"wsserver": { + "advertised_scheme": "wss", + "advertised_host": "mybigchaindb.com", + "advertised_port": 443 +} +``` + +**Default values (from a config file)** + +```js +"wsserver": { + "advertised_scheme": "ws", + "advertised_host": "localhost", + "advertised_port": 9985 +} +``` + +## log.* + +The `log.*` settings are to configure logging. + +**Example** + +```js +{ + "log": { + "file": "/var/log/bigchaindb.log", + "error_file": "/var/log/bigchaindb-errors.log", + "level_console": "info", + "level_logfile": "info", + "datefmt_console": "%Y-%m-%d %H:%M:%S", + "datefmt_logfile": "%Y-%m-%d %H:%M:%S", + "fmt_console": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "fmt_logfile": "%(asctime)s [%(levelname)s] (%(name)s) %(message)s", + "granular_levels": {} +} +``` + +**Default values** + +```js +{ + "log": { + "file": "~/bigchaindb.log", + "error_file": "~/bigchaindb-errors.log", + "level_console": "info", + "level_logfile": "info", + "datefmt_console": "%Y-%m-%d %H:%M:%S", + "datefmt_logfile": "%Y-%m-%d %H:%M:%S", + "fmt_logfile": "[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)", + "fmt_console": "[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)", + "granular_levels": {} +} +``` + +### log.file + +The full path to the file where logs should be written. +The user running `bigchaindb` must have write access to the +specified path. + +**Log rotation:** Log files have a size limit of about 200 MB +and will be rotated up to five times. +For example, if `log.file` is set to `"~/bigchain.log"`, then +logs would always be written to `bigchain.log`. Each time the file +`bigchain.log` reaches 200 MB it will be closed and renamed +`bigchain.log.1`. If `bigchain.log.1` and `bigchain.log.2` already exist they +would be renamed `bigchain.log.2` and `bigchain.log.3`. This pattern would be +applied up to `bigchain.log.5` after which `bigchain.log.5` would be +overwritten by `bigchain.log.4`, thus ending the rotation cycle of whatever +logs were in `bigchain.log.5`. + +### log.error_file + +Similar to `log.file` (see above), this is the +full path to the file where error logs should be written. + +### log.level_console + +The log level used to log to the console. Possible allowed values are the ones +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), +but case-insensitive for the sake of convenience: + +```text +"critical", "error", "warning", "info", "debug", "notset" +``` + +### log.level_logfile + +The log level used to log to the log file. Possible allowed values are the ones +defined by [Python](https://docs.python.org/3.6/library/logging.html#levels), +but case-insensitive for the sake of convenience: + +```text +"critical", "error", "warning", "info", "debug", "notset" +``` + +### log.datefmt_console + +The format string for the date/time portion of a message, when logged to the +console. + +For more information on how to construct the format string please consult the +table under [Python's documentation of time.strftime(format[, t])](https://docs.python.org/3.6/library/time.html#time.strftime). + +### log.datefmt_logfile + +The format string for the date/time portion of a message, when logged to a log + file. + +For more information on how to construct the format string please consult the +table under [Python's documentation of time.strftime(format[, t])](https://docs.python.org/3.6/library/time.html#time.strftime). + +### log.fmt_console + +A string used to format the log messages when logged to the console. + +For more information on possible formatting options please consult Python's +documentation on +[LogRecord attributes](https://docs.python.org/3.6/library/logging.html#logrecord-attributes). + +### log.fmt_logfile + +A string used to format the log messages when logged to a log file. + +For more information on possible formatting options please consult Python's +documentation on +[LogRecord attributes](https://docs.python.org/3.6/library/logging.html#logrecord-attributes). + +### log.granular_levels + +Log levels for Planetmint's modules. This can be useful to control the log +level of specific parts of the application. As an example, if you wanted the +logging of the `core.py` module to be more verbose, you would set the + configuration shown in the example below. + +**Example** + +```js +{ + "log": { + "granular_levels": { + "bichaindb.core": "debug" + } +} +``` + +**Default value** + +```js +{} +``` + +## tendermint.* + +The settings with names of the form `tendermint.*` tell Planetmint Server +where it can connect to the node's Tendermint instance. + +* `tendermint.host` is the hostname (FQDN)/IP address of the Tendermint instance. +* `tendermint.port` is self-explanatory. + +**Example using environment variables** + +```text +export PLANETMINT_TENDERMINT_HOST=tendermint +export PLANETMINT_TENDERMINT_PORT=26657 +``` + +**Default values** + +```js +"tendermint": { + "host": "localhost", + "port": 26657 +} +``` diff --git a/docs/root/source/installation/node-setup/deploy-a-machine.md b/docs/root/source/installation/node-setup/deploy-a-machine.md new file mode 100644 index 0000000..dc84990 --- /dev/null +++ b/docs/root/source/installation/node-setup/deploy-a-machine.md @@ -0,0 +1,64 @@ + + +# Deploy a Machine for Your Planetmint Node + +The first step is to deploy a machine for your Planetmint node. +It might be a virtual machine (VM) or a real machine, for example, +an EC2 on AWS or a droplet on Digital Ocean. +If you follow this simple deployment template, all your node's +software will run on that one machine. + +We don't make any assumptions about _where_ you run the machine. +It might be in Azure, AWS, your data center or a Raspberry Pi. + +## IP Addresses + +The following instructions assume all the nodes +in the network (including yours) have public IP addresses. +(A Planetmint network _can_ be run inside a private network, +using private IP addresses, but we don't cover that here.) + +## Operating System + +**Use Ubuntu 18.04 or Ubuntu Server 18.04 as the operating system.** + +Similar instructions will work on other versions of Ubuntu, +and other recent Debian-like Linux distros, +but you may have to change the names of the packages, +or install more packages. + +## Network Security Group + +If your machine is in AWS or Azure, for example, _and_ +you want users to connect to Planetmint via HTTPS, +then you should configure its network security group +to allow all incoming and outgoing traffic for: + +* TCP on port 22 (SSH) +* TCP on port 80 (HTTP) +* TCP on port 443 (HTTPS) +* Any protocol on port 26656 (Tendermint P2P) + +If you don't care about HTTPS, then forget about port 443, +and replace port 80 with port 9984 (the default Planetmint HTTP port). + +## Update Your System + +SSH into your machine and update all its OS-level packages: + +``` +sudo apt update +sudo apt full-upgrade +``` + +## DNS Setup + +* Register a domain name for your Planetmint node, such as `example.com` +* Pick a subdomain of that domain for your Planetmint node, such as `bnode.example.com` +* Create a DNS "A Record" pointing your chosen subdomain (such as `bnode.example.com`) + at your machine's IP address. diff --git a/docs/root/source/installation/node-setup/index.rst b/docs/root/source/installation/node-setup/index.rst new file mode 100644 index 0000000..08a3827 --- /dev/null +++ b/docs/root/source/installation/node-setup/index.rst @@ -0,0 +1,25 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Node setup +========== + +You can use the all-in-one docker solution, or install Tendermint, MongoDB, and Planetmint step by step. For more advanced users and for development, the second option is recommended. + +.. toctree:: + :maxdepth: 1 + + deploy-a-machine + aws-setup + all-in-one-bigchaindb + bigchaindb-node-ansible + set-up-node-software + set-up-nginx + configuration + bigchaindb-cli + troubleshooting + production-node/index + release-notes diff --git a/docs/root/source/installation/node-setup/production-node/index.rst b/docs/root/source/installation/node-setup/production-node/index.rst new file mode 100644 index 0000000..2b1300e --- /dev/null +++ b/docs/root/source/installation/node-setup/production-node/index.rst @@ -0,0 +1,17 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Production Nodes +================ + +.. toctree:: + :maxdepth: 1 + + node-requirements + node-assumptions + node-components + node-security-and-privacy + reverse-proxy-notes diff --git a/docs/root/source/installation/node-setup/production-node/node-assumptions.md b/docs/root/source/installation/node-setup/production-node/node-assumptions.md new file mode 100644 index 0000000..cc1aa8e --- /dev/null +++ b/docs/root/source/installation/node-setup/production-node/node-assumptions.md @@ -0,0 +1,25 @@ + + +# Production Node Assumptions + +Be sure you know the key Planetmint terminology: + +* [Planetmint node, Planetmint network and Planetmint consortium](https://docs.bigchaindb.com/en/latest/terminology.html) + +Note that there are a few kinds of nodes: + +- A **dev/test node** is a node created by a developer working on Planetmint Server, e.g. for testing new or changed code. A dev/test node is typically run on the developer's local machine. + +- A **bare-bones node** is a node deployed in the cloud, either as part of a testing network or as a starting point before upgrading the node to be production-ready. + +- A **production node** is a node that is part of a consortium's Planetmint network. A production node has the most components and requirements. + +We make some assumptions about production nodes: + +1. Each production node is set up and managed by an experienced professional system administrator or a team of them. +1. Each production node in a network is managed by a different person or team. diff --git a/docs/root/source/installation/node-setup/production-node/node-components.md b/docs/root/source/installation/node-setup/production-node/node-components.md new file mode 100644 index 0000000..44f2abe --- /dev/null +++ b/docs/root/source/installation/node-setup/production-node/node-components.md @@ -0,0 +1,30 @@ + + +# Production Node Components + +A production Planetmint node must include: + +* Planetmint Server +* MongoDB Server 3.4+ (mongod) +* Tendermint +* Storage for MongoDB and Tendermint + +It could also include several other components, including: + +* NGINX or similar, to provide authentication, rate limiting, etc. +* An NTP daemon running on all machines running Planetmint Server or mongod, and possibly other machines +* Probably _not_ MongoDB Automation Agent. It's for automating the deployment of an entire MongoDB cluster. +* MongoDB Monitoring Agent +* MongoDB Backup Agent +* Log aggregation software +* Monitoring software +* Maybe more + +The relationship between the main components is illustrated below. + +![Components of a production node](../../_static/Node-components.png) diff --git a/docs/root/source/installation/node-setup/production-node/node-requirements.md b/docs/root/source/installation/node-setup/production-node/node-requirements.md new file mode 100644 index 0000000..c54d11f --- /dev/null +++ b/docs/root/source/installation/node-setup/production-node/node-requirements.md @@ -0,0 +1,22 @@ + + +# Production Node Requirements + +**This page is about the requirements of Planetmint Server.** You can find the requirements of MongoDB, Tendermint and other [production node components](node-components) in the documentation for that software. + +## OS Requirements + +Planetmint Server requires Python 3.5+ and Python 3.5+ [will run on any modern OS](https://docs.python.org/3.5/using/index.html), but we recommend using an LTS version of [Ubuntu Server](https://www.ubuntu.com/server) or a similarly server-grade Linux distribution. + +_Don't use macOS_ (formerly OS X, formerly Mac OS X), because it's not a server-grade operating system. Also, BigchaindB Server uses the Python multiprocessing package and [some functionality in the multiprocessing package doesn't work on Mac OS X](https://docs.python.org/3.6/library/multiprocessing.html#multiprocessing.Queue.qsize). + +## General Considerations + +Planetmint Server runs many concurrent processes, so more RAM and more CPU cores is better. + +As mentioned on the page about [production node components](node-components), every machine running Planetmint Server should be running an NTP daemon. diff --git a/docs/root/source/installation/node-setup/production-node/node-security-and-privacy.md b/docs/root/source/installation/node-setup/production-node/node-security-and-privacy.md new file mode 100644 index 0000000..4841c94 --- /dev/null +++ b/docs/root/source/installation/node-setup/production-node/node-security-and-privacy.md @@ -0,0 +1,18 @@ + + +# Production Node Security & Privacy + +Here are some references about how to secure an Ubuntu 18.04 server: + +- [Ubuntu 18.04 - Ubuntu Server Guide - Security](https://help.ubuntu.com/lts/serverguide/security.html.en) +- [Ubuntu Blog: National Cyber Security Centre publish Ubuntu 18.04 LTS Security Guide](https://blog.ubuntu.com/2018/07/30/national-cyber-security-centre-publish-ubuntu-18-04-lts-security-guide) + +Also, here are some recommendations a node operator can follow to enhance the privacy of the data coming to, stored on, and leaving their node: + +- Ensure that all data stored on a node is encrypted at rest, e.g. using full disk encryption. This can be provided as a service by the operating system, transparently to Planetmint, MongoDB and Tendermint. +- Ensure that all data is encrypted in transit, i.e. enforce using HTTPS for the HTTP API and the Websocket API. This can be done using NGINX or similar, as we do with the IPDB Testnet. diff --git a/docs/root/source/installation/node-setup/production-node/reverse-proxy-notes.md b/docs/root/source/installation/node-setup/production-node/reverse-proxy-notes.md new file mode 100644 index 0000000..27e4ffe --- /dev/null +++ b/docs/root/source/installation/node-setup/production-node/reverse-proxy-notes.md @@ -0,0 +1,58 @@ + + +# Using a Reverse Proxy + +You may want to: + +* rate limit inbound HTTP requests, +* authenticate/authorize inbound HTTP requests, +* block requests with an HTTP request body that's too large, or +* enable HTTPS (TLS) between your users and your node. + +While we could have built all that into Planetmint Server, +we didn't, because you can do all that (and more) +using a reverse proxy such as NGINX or HAProxy. +(You would put it in front of your Planetmint Server, +so that all inbound HTTP requests would arrive +at the reverse proxy before *maybe* being proxied +onwards to your Planetmint Server.) +For detailed instructions, see the documentation +for your reverse proxy. + +Below, we note how a reverse proxy can be used +to do some Planetmint-specific things. + +You may also be interested in +[our NGINX configuration file template](https://github.com/bigchaindb/nginx_3scale/blob/master/nginx.conf.template) +(open source, on GitHub). + + +## Enforcing a Max Transaction Size + +The Planetmint HTTP API has several endpoints, +but only one of them, the `POST /transactions` endpoint, +expects a non-empty HTTP request body: +the transaction being submitted by the user. + +If you want to enforce a maximum-allowed transaction size +(discarding any that are larger), +then you can do so by configuring a maximum request body size +in your reverse proxy. +For example, NGINX has the `client_max_body_size` +configuration setting. You could set it to 15 kB +with the following line in your NGINX config file: + +```text +client_max_body_size 15k; +``` + +For more information, see +[the NGINX docs about client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). + +Note: By enforcing a maximum transaction size, you +[indirectly enforce a maximum crypto-conditions complexity](https://github.com/bigchaindb/bigchaindb/issues/356#issuecomment-288085251). diff --git a/docs/root/source/installation/node-setup/release-notes.md b/docs/root/source/installation/node-setup/release-notes.md new file mode 100644 index 0000000..609885e --- /dev/null +++ b/docs/root/source/installation/node-setup/release-notes.md @@ -0,0 +1,16 @@ + + +# Release Notes + +You can find a list of all Planetmint Server releases and release notes on GitHub at: + +[https://github.com/bigchaindb/bigchaindb/releases](https://github.com/bigchaindb/bigchaindb/releases) + +The [CHANGELOG.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CHANGELOG.md) contains much the same information, but it also has notes about what to expect in the _next_ release. + +We also have [a roadmap document in ROADMAP.md](https://github.com/bigchaindb/org/blob/master/ROADMAP.md). diff --git a/docs/root/source/installation/node-setup/set-up-nginx.md b/docs/root/source/installation/node-setup/set-up-nginx.md new file mode 100644 index 0000000..689c0b7 --- /dev/null +++ b/docs/root/source/installation/node-setup/set-up-nginx.md @@ -0,0 +1,43 @@ + + +# Set Up NGINX + +If you don't want HTTPS +(for communications between the external world and your node), +then you can skip all the NGINX steps on this page. + +Note: This simple deployment template uses NGINX for more than just HTTPS. +For example, it also does basic rate limiting. + +## Install NGINX + +SSH into your machine and install NGINX: + +``` +sudo apt update +sudo apt install nginx +``` + +## Configure & Reload NGINX + +Get an SSL certificate for your node's subdomain (such as `bnode.example.com`). + +* Copy the SSL private key into `/etc/nginx/ssl/cert.key` +* Create a "PEM file" (text file) by concatenating your SSL certificate with all intermediate certificates (_in that order, with the intermediate certs last_). +* Copy that PEM file into `/etc/nginx/ssl/cert.pem` +* In the + [bigchaindb/bigchaindb repository on GitHub](https://github.com/bigchaindb/bigchaindb), + find the file `nginx/nginx.conf` and copy its contents to + `/etc/nginx/nginx.conf` on your machine (i.e. replace the existing file there). +* Edit that file (`/etc/nginx/nginx.conf`): replace the two instances of + the string `example.testnet2.com` + with your chosen subdomain (such as `bnode.example.com`). +* Reload NGINX by doing: + ``` + sudo service nginx reload + ``` diff --git a/docs/root/source/installation/node-setup/set-up-node-software.md b/docs/root/source/installation/node-setup/set-up-node-software.md new file mode 100644 index 0000000..ee0e99d --- /dev/null +++ b/docs/root/source/installation/node-setup/set-up-node-software.md @@ -0,0 +1,112 @@ + + +# Set Up Planetmint, MongoDB and Tendermint + +We now install and configure software that must run +in every Planetmint node: Planetmint Server, +MongoDB and Tendermint. + +## Install Planetmint Server + +Planetmint Server requires **Python 3.6+**, so make sure your system has it. + +Install the required OS-level packages: + +``` +# For Ubuntu 18.04: +sudo apt install -y python3-pip libssl-dev +# Ubuntu 16.04, and other Linux distros, may require other packages or more packages +``` + +Planetmint Server requires [gevent](http://www.gevent.org/), and to install gevent, you must use pip 19 or later (as of 2019, because gevent now uses manylinux2010 wheels). Upgrade pip to the latest version: + +``` +sudo pip3 install -U pip +``` + +Now install the latest version of Planetmint Server. +You can find the latest version by going +to the [Planetmint project release history page on PyPI](https://pypi.org/project/Planetmint/#history). +For example, to install version 2.2.2, you would do: + +``` +# Change 2.0.0 to the latest version as explained above: +sudo pip3 install bigchaindb==2.2.2 +``` + +Check that you installed the correct version of Planetmint Server using `bigchaindb --version`. + +## Configure Planetmint Server + +To configure Planetmint Server, run: + +``` +bigchaindb configure +``` + +The first question is ``API Server bind? (default `localhost:9984`)``. + +* If you're using NGINX (e.g. if you want HTTPS), + then accept the default value (`localhost:9984`). +* If you're not using NGINX, then enter the value `0.0.0.0:9984` + +You can accept the default value for all other Planetmint config settings. + +If you're using NGINX, then you should edit your Planetmint config file +(in `$HOME/.bigchaindb` by default) and set the following values +under `"wsserver"`: + +``` +"advertised_scheme": "wss", +"advertised_host": "bnode.example.com", +"advertised_port": 443 +``` + +where `bnode.example.com` should be replaced by your node's actual subdomain. + +## Install (and Start) MongoDB + +Install a recent version of MongoDB. +Planetmint Server requires version 3.4 or newer. + +``` +sudo apt install mongodb +``` + +If you install MongoDB using the above command (which installs the `mongodb` package), +it also configures MongoDB, starts MongoDB (in the background), +and installs a MongoDB startup script +(so that MongoDB will be started automatically when the machine is restarted). + +Note: The `mongodb` package is _not_ the official MongoDB package +from MongoDB the company. If you want to install the official MongoDB package, +please see +[the MongoDB documentation](https://docs.mongodb.com/manual/installation/). +Note that installing the official package _doesn't_ also start MongoDB. + +## Install Tendermint + +The version of Planetmint Server described in these docs only works well +with Tendermint 0.31.5 (not a higher version number). Install that: + +``` +sudo apt install -y unzip +wget https://github.com/tendermint/tendermint/releases/download/v0.31.5/tendermint_v0.31.5_linux_amd64.zip +unzip tendermint_v0.31.5_linux_amd64.zip +rm tendermint_v0.31.5_linux_amd64.zip +sudo mv tendermint /usr/local/bin +``` + +## Start Configuring Tendermint + +You won't be able to finish configuring Tendermint until you have some information +from the other nodes in the network, but you can start by doing: + +``` +tendermint init +``` diff --git a/docs/root/source/installation/node-setup/troubleshooting.md b/docs/root/source/installation/node-setup/troubleshooting.md new file mode 100644 index 0000000..036046e --- /dev/null +++ b/docs/root/source/installation/node-setup/troubleshooting.md @@ -0,0 +1,95 @@ +# Troubleshooting + +## General Tips + +- Check the Planetmint, Tendermint and MongoDB logs. + For help with that, see the page about [Logging and Log Rotation](../appendices/log-rotation). +- Try Googling the error message. + +## Tendermint Tips + +* [Configure Tendermint to create no empty blocks](https://tendermint.com/docs/tendermint-core/using-tendermint.html#no-empty-blocks). +* Store the Tendermint data on a fast drive. You can do that by changing [the location of TMHOME](https://tendermint.com/docs/tendermint-core/using-tendermint.html#directory-root) to be on the fast drive. + +See the [Tendermint tips in the vrde/notes repository](https://github.com/vrde/notes/tree/master/tendermint). + +## Resolving Tendermint Connectivity Problems + +To check which nodes your node is connected to (via Tendermint protocols), do: + +```text +# if you don't have jq installed, then install it +sudo apt install jq +# then do +curl -s localhost:26657/net_info | jq ".result.peers[].node_info | {id, listen_addr, moniker}" +``` + +Note: Tendermint has other endpoints besides `/net_info`: see [the Tendermint RPC docs](https://tendermint.github.io/slate/?shell#introduction). + +If you're running your network inside a [private network](https://en.wikipedia.org/wiki/Private_network), e.g. with IP addresses of the form 192.168.x.y, then you may have to change the following setting in `config.toml`: + +```text +addr_book_strict = false +``` + +## Refreshing Your Node + +If you want to refresh your node back to a fresh empty state, then your best bet is to terminate it and deploy a new machine, but if that's not an option, then you can: + +* drop the `bigchain` database in MongoDB using `bigchaindb drop` (but that only works if MongoDB is running) +* reset Tendermint using `tendermint unsafe_reset_all` +* delete the directory `$HOME/.tendermint` + +## Shutting Down Planetmint + +If you want to stop/kill Planetmint, you can do so by sending `SIGINT`, `SIGQUIT` or `SIGTERM` to the running Planetmint +process(es). Depending on how you started Planetmint i.e. foreground or background. e.g. you started Planetmint in the background as mentioned above in the guide: + +```bash +$ nohup bigchaindb start 2>&1 > bigchaindb.log & + +$ # Check the PID of the main Planetmint process +$ ps -ef | grep bigchaindb + *
+ +
+ +
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docs/root/source/korean/_static/CREATE_and_TRANSFER_example.png b/docs/root/source/korean/_static/CREATE_and_TRANSFER_example.png new file mode 100644 index 0000000..f9ef1ee Binary files /dev/null and b/docs/root/source/korean/_static/CREATE_and_TRANSFER_example.png differ diff --git a/docs/root/source/korean/_static/CREATE_example.png b/docs/root/source/korean/_static/CREATE_example.png new file mode 100644 index 0000000..88a933c Binary files /dev/null and b/docs/root/source/korean/_static/CREATE_example.png differ diff --git a/docs/root/source/korean/_static/schemaDB.png b/docs/root/source/korean/_static/schemaDB.png new file mode 100644 index 0000000..4ed7ad7 Binary files /dev/null and b/docs/root/source/korean/_static/schemaDB.png differ diff --git a/docs/root/source/korean/assets_ko.md b/docs/root/source/korean/assets_ko.md new file mode 100644 index 0000000..ad49993 --- /dev/null +++ b/docs/root/source/korean/assets_ko.md @@ -0,0 +1,21 @@ + + + +BigchainDB가 자산 등록 및 전송에 적합한 방법 +========================================================== + +BigchainDB는 모든 종류의 데이터를 저장할 수 있지만 자산 등록 및 전송을 저장하는 데 특히 유용합니다.: + +* Planetmint 네트워크에 전송되어 체크되고 저장되는 (있는 경우) 트랜잭션은 기본적으로 CREATE 트랜잭션과 TRANSFER 트랜잭션의 두 가지가 있습니다. +* CREATE 트랜잭션은 임의의 메타 데이터와 함께 모든 종류의 자산 (나눌 수 없거나 분할 할 수없는)을 등록하는 데 사용할 수 있습니다. +* 저작물에는 0 명, 1 명 또는 여러 명의 소유자가있을 수 있습니다. +* 자산 소유자는 자산을 신규 소유자에게 양도하려는 사람이 만족해야하는 조건을 지정할 수 있습니다. 예를 들어 5 명의 현재 소유자 중 최소 3 명이 TRANSFER 트랜잭션에 암호를 사용해야합니다. +* BigchainDB는 TRANSFER 트랜잭션의 유효성을 검사하는 과정에서 조건이 충족되었는지 확인합니다. (또한 누구나 만족하는지 확인할 수 있습니다.) +* BigchainDB는 자산의 이중 지출을 방지합니다. +* 유효성이 검증 된 트랜잭션은 [변경불가능](https://github.com/bigchaindb/bigchaindb/blob/master/docs/root/source/korean/immutable-ko.md) 입니다. + + Note + + 우리는 "소유자"라는 단어를 다소 느슨하게 사용했습니다. **보다 정확한 단어**는 이행자, 서명자, 조정자 또는 이전 가능 요소 일 수 있습니다. 관련 [Planetmint Transaction Spec](https://github.com/bigchaindb/BEPs/tree/master/tx-specs/)의 Owners에 대한 참고 사항 절을 참조하십시오. diff --git a/docs/root/source/korean/bft-ko.md b/docs/root/source/korean/bft-ko.md new file mode 100644 index 0000000..22fee0c --- /dev/null +++ b/docs/root/source/korean/bft-ko.md @@ -0,0 +1,13 @@ + + +# BigchainDB와 Byzantine Fault Tolerance + +[Planetmint Server](https://docs.bigchaindb.com/projects/server/en/latest/index.html) +는 블록체인 합의와 트랜잭션 복제에 [Tendermint](https://tendermint.com/)를 사용합니다. + +그리고 Tendermint 는 [Byzantine Fault Tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance). diff --git a/docs/root/source/korean/decentralized_kor.md b/docs/root/source/korean/decentralized_kor.md new file mode 100644 index 0000000..c597f77 --- /dev/null +++ b/docs/root/source/korean/decentralized_kor.md @@ -0,0 +1,24 @@ + + +# Planetmint 분산 방식 + +분산이란 모든 것을 소유하거나 통제하는 사람이 없고, 단 하나의 실패 지점이 없다는 것을 의미합니다. + +이상적으로, Planetmint 네트워크에서 각각의 노드는 다른 개인 또는 조직에 의해 소유되고 관리됩니다. 네트워크가 한 조직 내에 상주하고 있더라도, 각 노드를 다른 사용자나 부서로 제어하는 것이 좋습니다. + +우리는 "Planetmint 컨소시엄" (또는 단지 "컨소시엄")은 Planetmint 네트워크의 노드를 구동하는 사람들 혹은 조직을 의미합니다. 컨소시엄은 회원제나 정책과 같은 결정을 내리기 위한 어떠한 형태의 거버넌스 요구합니다. 거버넌스 프로세스의 정확한 세부사항은 각 컨소시엄에 의해 결정되지만, 상당히 분산될 수 있습니다. + +컨소시엄은 관할의 다양성과 지리적 다양성 및 기타 종류의 다양성을 증가시킴으로써 분산화(및 탄력성)를 증가시킬 수 있습니다. 이 아이디어는 [노드 다양성 부문](diversity-ko)에서 확장됩니다. + +Planetmint 네트워크에는 오래된 특정한 위치를 가지는 노드가 없습니다. 모든 노드들은 동일한 소프트웨어를 실행하고 동일한 작업을 수행합니다. + +만약 노드에 대한 관리자 접근 권한이 있는 경우, 해당 노드를 제거할 수 있지만(예: 해당 노드에 저장된 데이터 변경 또는 삭제), 이러한 변경 사항은 해당 노드에 분리된 상태로 유지되어야 합니다. Planetmint 네트워크는 노드의 3분의 1 이상이 손상된 경우에만 손상될 수 있습니다. 자세한 내용은 [Tendermint 문서](https://tendermint.com/docs/introduction/introduction.html)을 참조하십시오. + +노드의 관리자나 슈퍼 유저도 자산을 전송할 수 없다는 점에 유의하십시오. 유효한 전송 트랜잭션을 생성하는 유일한 방법은 자산에 대한 현재 암호화 조건을 충족하는 것이며 관리자/슈퍼사용자는 필요한 정보를 가지고 있지 않기 때문에 이 작업을 수행할 수 없습니다(예: 개인 키). + +노드의 관리자나 슈퍼 사용자도 자산을 전송할 수는 없다는 점을 유의하십시오. 타당한 전송 트랜잭션을 만드는 유일한 방법은 자산에 대한 현재 암호화 조건을 충족시키는 것이며, 관리자 또는 슈퍼 사용자는 필요한 정보를 가지고 있지 않기 때문에 이 작업을 수행할 수 없습니다. (예: 개인 키) \ No newline at end of file diff --git a/docs/root/source/korean/diversity-ko.md b/docs/root/source/korean/diversity-ko.md new file mode 100644 index 0000000..15313b4 --- /dev/null +++ b/docs/root/source/korean/diversity-ko.md @@ -0,0 +1,18 @@ + + +# 노드 다양성의 종류 + + +한 명의 유저나 이벤트가 노드의 "상당수" 부분을 제어하거나 손상시키는 것을 어렵게 만드는 절차가 수행되어야 합니다.(Planetmint 서버는 Tendermint를 사용하기 때문에 여기서 "상당수"는 노드의 1/3을 말합니다.) 아래에 쓰여진 여러 가지의 다양성을 고려해야 합니다. 모든 종류에 있어서 높은 다양성을 갖는 것은 꽤 어려운 일입니다. + +1. **관할권 다양성.** 노드는 여러 합법적 관할권 내의 참여자들이 제어합니다. 이는 어떤 일을 수행하기에 이 수단들을 사용하기 어렵게 할 수 있습니다. +1. **지리적 다양성.** 서버는 지리적으로 여러 곳에 물리적으로 위치합니다. 이는 자연 재해(홍수 또는 지진 등)가 문제를 야기할 만큼 손상되기 어렵게 합니다. +1. **호스팅 다양성.** 서버는 여러 호스팅 공급자(ex. Amazon Web Services, Microsoft Azure, Digital Oceure, Rackspace)가 호스팅해야 합니다. 이는 하나의 호스팅 공급자가 '상당 수'의 노드에 영향을 미치기가 어려워집니다. +1. **일반적인 의미의 다양성.** 일반적으로 멤버십 다양성은 컨소시엄에 많은 이점을 줍니다. 예를 들어, 그것은 문제 해결에 필요한 다양한 아이디어 소스를 컨소시엄에 제공합니다. + +참고: 모든 노드가 동일한 코드(ex. BigchainDB의 동일한 구현)를 실행하고 있는 경우, 해당 코드의 버그를 사용하여 모든 노드를 손상시킬 수 있습니다. 이상적으로는 Planetmint 서버(예: Python, Go 등)에서 운영되고 있는 다양한 구현이 있어, 컨소시엄은 다양한 서버 구현을 할 수 있을 것입니다. 운영 체제에 대해서도 이는 유사하게 적용됩니다. diff --git a/docs/root/source/korean/immutable-ko.md b/docs/root/source/korean/immutable-ko.md new file mode 100644 index 0000000..7c92938 --- /dev/null +++ b/docs/root/source/korean/immutable-ko.md @@ -0,0 +1,27 @@ + + +# 어떻게 BigchainDB는 불변성을 갖는가 + +*Imunable*이라는 단어는 "시간 경과에 따른 불변성"을 의미합니다. 예를 들어, π의 10진수 값은 변경할 수 없습니다(3.14159...). + +블록체인 커뮤니티는 종종 블록체인을 "불변하다"고 설명합니다. 우리가 그 단어를 문자 그대로 해석한다면, 그것은 블록체인 데이터가 변경할 수 없거나 영구적이라는 것을 의미하는데, 이것은 말이 안됩니다. 데이터는 *변경 될 수 있습니다.* 예를 들어, 전염병이 인류를 멸종 시킬 수도 있는 것처럼 데이터는 수분에 의한 손상, 온도에 의한 손상, 엔트로피의 일반적인 증가로 인해 시간이 지남에 따라 손상될 수 있습니다. + +블록체인 데이터가 일반적인 경우보다 변경(혹은 삭제)하기가 더 어려운 것은 사실입니다. 블록체인 데이터는 단순히 (의도적인) "변형 방지"에 그치지 않고 하드 드라이브의 데이터 손상과 같은 비의도적으로 발생할 수 있는 무작위 변경에도 대응합니다. 따라서 블록체인에서 "불변한다"라는 단어를 우리는 어떤 모든 의도와 목적이 *실제적으로* 불변한 것으로 해석합니다. (언어학자들은 "불변한다"라는 단어가 블록체인 커뮤니티에서 쓰이는 *기술적 용어*라고 말할 것입니다.) + +블록체인 데이터는 여러 가지 방법을 통해 불변성을 가질 수 있습니다: + +1. **데이터 변경 또는 삭제를 위한 API 없음.** Blockchain 소프트웨어는 일반적으로 블록체인에 저장된 데이터를 변경하거나 삭제하기 위한 API를 제공하지 않습니다. Planetmint 역시 관련한 API를 제공하지 않습니다 . 이것은 변경이나 삭제가 *다른 방식*으로 일어나는 것을 막지 못합니다. 이것은 단지 하나의 방어선일 뿐입니다. +1. **복제.** 모든 데이터는 여러 곳에 복제(복사)됩니다. 복제 팩터가 높을수록, 모든 복제본을 변경하거나 삭제하기가 더 어려워집니다. +1. **내부 감시 장치.** 모든 노드가 모든 변경 사항을 모니터링하고 허용되지 않은 변경 사항이 발생하면 적절한 조치를 취할 수 있습니다. +1. **외부 감시 장치.** 컨소시엄은 부정행위를 찾아 데이터를 모니터링하고 감사할 수 있는 검증된 제 3자를 선택할 수 있습니다. 공개적으로 읽을 수 있는 데이터를 가진 컨소시엄의 경우, 대중은 감사자 역할을 할 수 있습니다. +1. **경제적 인센티브.** 일부 블록체인 시스템은 기존의 저장된 데이터를 변경하는 데 많은 비용이 들게 합니다. 그 예로 작업 증명 및 지분 증명 시스템이 있습니다. BigchainDB의 경우에는 이런 인센티브를 사용하지 않습니다. +1. 변화에 대한 손쉬운 실행 취소를 위해 오류 수정 코드와 같은 고급 기술을 사용해 데이터를 저장할 수 있습니다 +1. **암호화폐의 표식**은 종종 메시지(예: 트랜잭션)가 도중에 손상되었는지 확인하고 메시지에 서명한 사용자를 확인하는 방법으로 사용됩니다. BigchainDB에서는 각 트랜잭션에 한 명 이상의 당사자가 서명해야 합니다 +1. **전체 또는 부분적 백업**은 때때로 자기 테이프 저장소, 기타 블록체인, 인쇄물 등에 기록될 수 있습니다. +1. **강력한 보안** 노드 소유자는 강력한 보안 정책을 채택하고 적용할 수 있습니다. +1. **노드 다양성.** 다양성으로 인해서 한 가지 요소(예: 자연 재해 또는 운영 체제 버그)가 상당 수의 노드를 손상시킬 수 없도록 합니다. [노드 다양성의 종류에 대한 섹션](https://github.com/bigchaindb/bigchaindb/blob/master/docs/root/source/korean/diversity-ko.md)을 참조하세요. diff --git a/docs/root/source/korean/index.rst b/docs/root/source/korean/index.rst new file mode 100644 index 0000000..c74430d --- /dev/null +++ b/docs/root/source/korean/index.rst @@ -0,0 +1,98 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Planetmint 문서 +======================== + +블록체인 데이터베이스인 BigchainDB를 만나보세요. + +`분산형 `_, `불변성 `_ 및 `자산에 대한 네이티브 지원 `_ 을 포함한 일부 데이터베이스 특성들과 블록체인 특성을 가지고 있습니다. + +높은 수준에서, 사용자는 Planetmint HTTP API를 사용하는 Planetmint 네트워크(노드 집합) 또는 Planetmint 파이썬 드라이버와 같은 API용 래퍼로 통신할 수 있습니다. 각 Planetmint 노드는 Planetmint Server 및 다양한 다른 소프트웨어를 실행합니다. 더 자세한 사항은 용어 페이지에서 이러한 용어 중 일부를 설명합니다. + +.. raw:: html + + + + + + + + + + + +BigchainDB에 대한 추가 정보 +------------------------------------------------------- + +.. toctree:: + :maxdepth: 1 + + Planetmint Docs Home + production-ready_kor + terminology_kor + decentralized_kor + diversity-ko + immutable-ko + bft-ko + query-ko + assets_ko + smart-contracts_ko + transaction-concepts_ko + store-files_ko + permissions-ko + private-data-ko + Data Models diff --git a/docs/root/source/korean/permissions-ko.md b/docs/root/source/korean/permissions-ko.md new file mode 100644 index 0000000..5e32438 --- /dev/null +++ b/docs/root/source/korean/permissions-ko.md @@ -0,0 +1,59 @@ + + +# Planetmint 사용 권한 + +BigchainDB를 사용하면 다른 사용자가 할 수 있는 것을 어느 정도 제어할 수 있습니다. +이 능력은 \*nix환경에서의 "권한", SQL에서의 "특권", 보안 환경에서의 "액세스 제어"와 유사합니다. + +## 출력 지출/이전 권한 + +BigchainDB에서, 모든 출력에는 연관된 조건(crypto-condition)이 있습니다. + +사용되지 않은 출력을 쓰거나 전송하려면, 사용자(또는 사용자 그룹)이 조건을 충족시켜야 합니다. +특정 사용자만이 출력을 보낼 권한이 있다는 뜻입니다. 가장 단순한 조건은, "공용 키에 해당하는 개인 키를 가진 사람만이 출력을 보낼 수 있습니다." 훨씬 더 정교한 조건들도 가능합니다, 예를 들어 “이 출력을 사용하려면,…" + +- "…회계 그룹의 모든 사람이 서명 할 수 있습니다." +- "…네 명 중 세 명이 서명해야 합니다." +- "…Bob이 반드시 서명해야 하거나 Tom과 Sylvia 둘 모두가 서명해야 합니다." + +자세한 내용은, [Planetmint Transactions Spec](https://github.com/bigchaindb/BEPs/tree/master/tx-specs/)관련 **트랜잭션 구성요소:조건** 섹션을 참조하세요. + +출력이 한번 소비되면 다시 사용할 수 없습니다: *아무도* 그렇게 할 권한이 없습니다. 즉, BigchainDB는 누구나 출력을 "이중 소비" 하도록 허용 하지 않습니다. + +## 쓰기 권한 + +누군가 TRANSFER 트랜잭션을 만들면, `metadata` 필드에 임의의 JSON 객체를 넣을 수 있다. (적정 범위 내에서; 실제 Planetmint 네트워크는 트랜잭션의 크기에 제한을 둔다.) 즉, TRANSFER 트랜잭션에서 원하는 모든 것을 쓸 수 있다. + +BigchainDB에서 "쓰기 권한"이 없다는 의미인가요? 아닙니다!! + +TRANSFER 트랜잭션은 입력이 이전 출력을 충족시키는 경우에만 유효(허용)합니다. 이 출력들에 대한 조건은 누가 유효한 TRANSFER 트랜잭션을 할 수 있는지 조절 할 것입니다. 즉, 출력에 대한 조건은 특정 사용자에게 관련 자산 내역에 무엇인가 쓸 수 있는 "쓰기 권한"을 부여하는 것과 같습니다. + +예를 들어, 당신은 BigchainDB를 사용하여 오직 당신만이 쓰기권한이 있는 공용 저널을 작성 할 수 있습니다. 방법은 다음과 같습니다: 먼저 하나의 출력으로 `asset.data` 을 통해 `{"title": "The Journal of John Doe"}` 와 같이 되도록 CREATE 트랜잭션을 생성합니다. 이 출력에는 금액 1과 사용자(개인 키를 가진)만이 출력을 보낼 수 있는 조건이 있습니다. 저널에 무엇인가를 추가하고 싶을 때마다, `metadata` 같은 필드에 최신 항목을 넣은 TRANSFER 트랜잭션을 새로 만들어야 합니다. + +```json +{"timestamp": "1508319582", + "entry": "I visited Marmot Lake with Jane."} +``` + +TRANSFER 트랜잭션에는 하나의 출력이 있습니다. 이 출력에는 금액1과 사용자(개인키를 가진)만이 출력을 보낼 수 있는 조건이 있습니다. 기타 등등. 당신만이 자산 내역(당신의 저널)에 덧붙일 수 있습니다. + +이와 같은 기술은 공학 노트북,공급망 기록,정부 회의록 등에도 사용 될 수 있습니다. + +또한 더 정교한 것들도 할 수 있습니다. 예를 들어, 누군가가 TRANSFER 트랜잭션을 작성할 때마다, *다른 누군가*에게 사용 권한을 부여하여 일종의 작성자-전달 혹은 연쇄 편지를 설정한다. + +Note + +누구나 CREATE 트랜잭션의 `asset.data` 필드에 있는 JSON(조건하에)을 쓸 수 있습니다. 허가가 필요하지 않습니다. + +## 읽기 권한 + +다음 페이지를 참고하세요, [:doc:Planetmint, Privacy and Private Data](https://github.com/bigchaindb/bigchaindb/blob/master/docs/root/source/korean/private-data-ko.md). + +## 역할 기반 액세스 제어(RBAC) + +2017년 9월에, 우리는 [Planetmint RBAC 하부 시스템을 정의 할 수 있는 방법에 대한 블로그 게시물](https://blog.bigchaindb.com/role-based-access-control-for-bigchaindb-assets-b7cada491997)을 게재 했습니다. 글을 쓴 시점(2018년 1월)에는 플러그인을 사용해야 해서, 표준 BigchainDB다음에서 사용가능한 [Planetmint Testnet](https://testnet.bigchaindb.com/) 를 사용 할 수 없었습니다. 이는 미래에 바뀔 수 있습니다. 만약 관심이 있다면, [BigchainDB로 연락하십시요.](https://www.bigchaindb.com/contact/) diff --git a/docs/root/source/korean/private-data-ko.md b/docs/root/source/korean/private-data-ko.md new file mode 100644 index 0000000..c0a176d --- /dev/null +++ b/docs/root/source/korean/private-data-ko.md @@ -0,0 +1,102 @@ + + +# Planetmint, 개인정보 및 개인 데이터 + +## 기본 정보 + +1. 한도 내에서 Planetmint 네트워크에 임의의 데이터(암호화 된 데이터 포함)를 저장 할 수 있습니다. 모든 트랜잭션에는 거의 모든 유니코드 문자열(최대 길이까지)을 저장 할 수 있는 `metadata` 섹션이 있습니다. 마찬가지로, 모든 CREATE 트랜잭션에는 거의 모든 유니코드 문자열을 저장 할 수 있는 `asset.data` 섹션이 있습니다. +2. 특정 Planetmint 거래 필드에 저장된 데이터는 암호화 해서는 안됩니다, 예를 들어 공용키 및 자산과 같이. BigchainDB는 Zcoin과 비슷한 개인 거래를 제공하지 않습니다. +3. 데이터가 BigchinDB 네트워크에 저장되면 변경 또는 삭제 될 수 없다고 가정하는 것이 좋습니다. +4. Planetmint 네트워크의 모든 노드에는 저장된 모든 데이터의 전체 복사본이 있습니다. +5. Planetmint 네트워크의 모든 노드는 저장된 모든 데이터를 읽을 수 있습니다. +6. Planetmint 노드(예를 들어 노드이 sysadmin)에 대한 전체 액세스 권한을 가진 모든 사용자는해당 노드에 저장된 모든 데이터를 읽을 수 있습니다. +7. Planetmint HTTP API를 통해 노드에 접근하는 모든 사용자는 BigchainDB에 저장된 모든 데이터를 찾고 읽을 수 있습니다. 액세스 권한이 있는 사람들의 목록은 매우 짧을 수 있습니다. +8. 외부 사용자와 Planetmint 노드 사이의 연결이(예를 들어 HTTPS를 사용하여) 암호화되 않으면도청자는 전송중인 모든 HTTP 요청 및 응답을 읽을 수 있습니다. +9. 만약 누군가가 평문에 접근 할 수 있다면(어디에서 가져왔는지 관계없이), 원칙적으로 이것을 전 세계와 공유 할 수 있습니다. 그렇게 하는 것을 어렵게 만들 수 있습니다, 예를 들어 데이터가 많고 방을 나갈 떄 검색되는 안전한 방 안에만 들어 갈 수 있는 것과 같습니다. + +## 오프 체인에서 개인 데이터 저장 + +시스템은 제3자 데이터베이스, 문서 저장소 또는 CMS(컨텐츠 관리 시스템)와 같은 오프 체인 데이터를 저장할 수 있으며, BigchinDB를 사용하여 다음 작업을 수행할 수 있습니다: + +- 제3자 시스템에 읽기 권한 또는 기타 권한이 있는 사용자를 추적합니다. 이 작업을 수행하는 방법의 예는 아래에 있습니다. +- 제3자 시스템에 대한 모든 요청을 영구적으로 기록합니다. +- 모든 문서의 변경 사항을 감지 할 수 있도록, 다른 곳에 저장된 문서의 해시를 저장합니다. +- 암호화 된 터널을 설정했다는 것을 증명할 수 있도록 두 개의 오프 체인 파티(예:Diffie-Hellman 키 교환) 간의 모든 핸드셰이크 설정 요청 및 응답을 기록합니다(독자가 해당 터널에 액세스하지 않고). 이 아이디어에 대한 자세한 내용은 [the Planetmint Privacy Protocols 저장소](https://github.com/bigchaindb/privacy-protocols)에 있습니다. + +특정 문서에 대한 읽기 권한을 가진 사람을 기록하는 간단한 방법은 제 3자 시스템(“Docpile“)이 모든 문서+사용자 쌍에 대해 BigchinDB 네트워크에 CREATE 트랜잭션을 저장하여 해당 사용자가 그 문서에 대한 읽기 권한을 가지고 있음을 나타낼 수 있습니다. 트랜잭션은 Docpile에 의해 서명 될 수 있습니다(또는 문서 소유자에 의해). 자산 데이터 필드는 1)사용자의 고유 ID 및 2)문서의 고유 ID를 포함합니다. CREATE 트랜잭션의 한 출력은 DocPile(또는 문서 소유자)에 의해서만 전송/소비 될 수 있습니다. + + +읽기 권한을 취소하기 위해, DocPile은 원래 사용자가 더 이상 해당 문서에 대한 읽기 권한을 가지고 있지 않다고 하는 메타 데이터 필드를 사용하여, 원래의 CREATE 트랜잭션에서 하나의 출력을 보내기 위한 TRANSFER 트랜잭션을 생성 할 수 있습니다. + +이는 무한정으로 수행될 수 있습니다,즉.사용자가 다시 읽기 권한을 가지고 있음을 나타내기 위해 다른 TRANSFER 트랜잭션을 DocPile에서 작성할 수 있습니다. + +DocPile은 CREATE → TRANSFER → TRANSFER → 사용자+문서 쌍에 대한 etc.chain 과정에서 사용자의 마지막 트랜잭션을 읽음으로써 주어진 문서에 대한 읽기 권한을 가지고 있는지 파악할 수 있습니다. + +여기에 같은 일을 하는 다른 방법들이 있다. 위는 단지 하나의 예시이다. + +위의 예시에서는 사용자가 소유한(통제 된)자산으로 “읽기 권한“을 취급하지 않았다는 것을 알 수 있습니다, 왜냐하면 사용 권한 자산이 사용자에게 주어 지면(사용자에 의해 양도되거나 사용자에 의해 생성된 경우) 사용자가 다시 Docpile로 전송 할 때까지 어떠한 것도 제어 할 수 없기 때문입니다(Docpile에 의해). + +## 체인에서 암호화 된 개인 데이터 저장 + +체인상에서 개인 데이터를 암호화하여 저장하는 방법에는 여러 가지가 있습니다. 모든 유스 케이스에는 고유한 목표와 제약이 있으며, 최상의 해결책은 유스 케이스에 달려있다. +[Planetmint 컨설팅 팀](https://www.bigchaindb.com/services/), 우리의 파트너와 함께, 당신의유스 케이스에 가장 적합한 솔루션을 설계하는 데 도움을 줄 수 있습니다. + +아래에서는 다양한 암호화 기본 설정을 사용하여 가능한 시스템을 설정하는 예제를 설명합니다. + +참고 사항: + +- Ed25519 키 쌍은 [메시지 암호화 및 암호 해독이 아닌](https://crypto.stackexchange.com/questions/27866/why-curve25519-for-encryption-but-ed25519-for-signatures) 암호화 서명 및 확인을 위해 설계되었습니다. 암호화의 경우, X25519와 같은 암호화를 위해 설계된 키 쌍을 사용해야 합니다. +- 누군가(또는 어떤 그룹)이 체인상의 암호화 된 데이터를 해독하는 방법을 발표하면 암호화 된 데이터에 액세스 할 수 있는 모든 사람이 평문을 가져올 수 있습니다. 데이터는 삭제할 수 없습니다. +- 암호화 된 데이터는 MongoDM에서 색인을 생성하거나 검색 할 수 없습니다.(암호문을 색인화하고 검색 할 수 있지만 유용하지는 않습니다.) 암호화 된 데이터를 색인화하고 검색하기 위해 준 유사 암호를 사용할 수 있지만, MongoDB는 이를 지원할 계획이 없습니다. 색인화 또는 키워드 검색이 필요한 경우 `asset.data`의 몇가지 필드 또는 `metadata`객체를 일반 텍스트로 남겨두고 민감한 정보를 암호화 된 하위 객체에 저장할 수 있습니다. + +### 시스템 예시 1 + +대칭 키로 데이터를 암호화하고 체인에(`metadata` 또는 `asset.data` 에서) 암호문을 저장하십시오. 키를 제 3자에게 알리려면, 공용 키를 사용하여 대칭 키를 암호화하고 암호화 키를 보냅니다. 개인 키로 대칭 키의 암호를 해독한 다음 대칭 키를 사용하여 on-chain 암호문의 암호를 해독할 수 있습니다. + +공용 키/ 개인 키 쌍과 함께 대칭 키를 사용하는 이유는 암호문을 한 번만 저장하면 되기 때문입니다. + +### 시스템 예시 2 + +이 예시에서는 [프록시 재-암호화](https://en.wikipedia.org/wiki/Proxy_re-encryption) 를 사용합니다: + +1. MegaCorp는 자체 공용 키를 사용하여 일부 데이터를 암호화 한 후 암호화 된 데이터(암호문1)을 Planetmint 네트워크에 저장합니다. + +2. MegaCorp는 다른 사람들이 암호화 된 데이터를 읽을 수 있게 하고 싶지만, 공용 키를 공유하지 않고 모든 새로운 수신자에 대해 스스로를 다시 암호화 할 필요가 없습니다. 대신 프록시 재 암호화 서비스를 제공하기 위해 Moxie라는 “프록시“를 찾습니다. +3. Zorban은 MegaCorp에 연결하여 데이터 읽기 권한을 요청합니다. +4. MegaCorp는 Zorban에게 공용 키를 요청합니다. +5. MegaCorp “재 암호화 키“를 생성하여 프록시 Moxie로 전송합니다. +6. Moxie (프록시)는 재 암호화 키를 사용하여 암호문 1을 암호화하고 암호문 2를 만듭니다. +7. Moxie는 Zorban(또는 Zorban에게 전달하는 MegaCorp)에게 암호문 2를 보냅니다. +8. Zorban은 개인 키를 사용하여 암호문 2를 해독해서 원본 암호화되지 않은 데이터를 가져옵니다. + +참고: + +- 프록시는 암호문만 볼 수 있습니다. 암호화 되지 않은 데이터는 볼 수 없습니다. +- Zorban은 암호문 1, 즉 체인 상의 데이터를 해독 할 수 있는 능력이 없습니다. +- 위의 흐름에는 다양한 변형이 있습니다. + +## 시스템 예시 3 + +이 예시는 [삭제 코딩](https://en.wikipedia.org/wiki/Erasure_code)을 사용합니다: + +1. 데이터를 n개의 조각으로 삭제하십시오. +2. 서로 다른 암호화 키로 n개의 조각을 암호화 하십시오. +3. n 개의 암호화 된 부분을 체인에 저장합니다 (예: n개의 별도 트랜잭션). +4. n 개의 암호 해독 키 각각을 다른 당사자와 공유하십시오. + +만약 k< N 인 키홀더가 k개의 조각들을 가져와서 해독한다면, 그것들은 원본 텍스트를 다시 만들 수 있습니다. k미만이면 충분하지 않습니다. + +### 시스템 예시 4 + +이 설정은 특수 노드가 데이터의 일부를 볼 수 있어야 하지만, 다른 노드는 볼 수 없어야 하는 기업용 블록 체인 시나리오에서 사용할 수 있습니다. + +- 특수 노드는 X25519 키 쌍 (또는 유사한 비대칭 *암호화*키 쌍)을 생성합니다 . +- Planetmint 최종 사용자는 특수 노드의 X25519 공용 키(암호화 키)를 찾습니다. + -최종 사용자는 위에서 언급 한 공용 키를 사용하여, asset.data 또는 메타 데이터(또는 모두)를 사용하여 유효한 Planetmint 트랜잭션을 생성합니다. +- 이는 asset.data 또는 메타 데이터의 내용이 유효성 검증에 중요하지 않은 트랜잭션에 대해서만 수행되므로, 모든 노드 운영자가 트랜잭션을 검증 할 수 있습니다. +- 특수 노드는 암호화 된 데이터를 해독 할 수 있지만, 다른 노드 운영자와 다른 최종 사용자는 할 수 없습니다. diff --git a/docs/root/source/korean/production-ready_kor.md b/docs/root/source/korean/production-ready_kor.md new file mode 100644 index 0000000..cbde174 --- /dev/null +++ b/docs/root/source/korean/production-ready_kor.md @@ -0,0 +1,12 @@ + + +# 배포 - 준비 + +경우에 따라, BigchainDB는 배포-준비가 될 수도 있고 되지 않을 수도 있습니다. 서비스 공급자에게 문의해야 합니다. 만약 BigchainDB를 (배포로) 전환하고자 한다면, 서비스 공급자에게 문의하십시오. + +참고 : BigchainDB는 "보증 없음" 섹션을 가지는 오픈소스 라이센스이며, 이는 전형적인 오픈소스 라이센스입니다. 이는 소프트웨어 산업의 표준입니다. 예를 들어, 리눅스 커널은 라이센스에 "보증 없음" 섹션을 가지고 있지만, 수십억 대의 시스템에 의해 배포되어 사용됩니다. 보증은 대개 서비스 공급자가 소프트웨어 라이센스 수준 이상으로 제공합니다. diff --git a/docs/root/source/korean/query-ko.md b/docs/root/source/korean/query-ko.md new file mode 100644 index 0000000..bf6f140 --- /dev/null +++ b/docs/root/source/korean/query-ko.md @@ -0,0 +1,202 @@ + + +Planetmint 쿼리 +=================== + +노드 operator는 MongoDB의 쿼리 엔진의 최대 성능을 사용하여 모든 트랜잭션, 자산 및 메타데이터를 포함하여 저장된 모든 데이터를 검색하고 쿼리할 수 있습니다. 노드 operator는 외부 사용자에게 얼마나 많은 쿼리 파워를 송출할지 스스로 결정할 수 있습니다. + + +예제 쿼리가 포함된 블로그 게시물 +------------------------------ + + +Planetmint 블로그에 MongoDB 도구를 사용하여 Planetmint 노드의 MongoDB 데이터베이스를 쿼리하는 방법에 대한 게시물을 올렸습니다. 데이터에 대한 일부 특정 예제 쿼리가 주요 내용입니다. [여기서 확인하세요](https://blog.bigchaindb.com/using-mongodb-to-query-bigchaindb-data-3fc651e0861b) + +MongoDB에 연결하기 +------------------------- + + +MongoDB 데이터베이스를 쿼리하려면 먼저 데이터베이스에 연결해야 합니다. 그러기 위해선 호스트 이름과 포트를 알아야 합니다. + +개발 및 테스트를 위해 지역 컴퓨터에서 Planetmint 노드를 실행 중인 경우 호스트 이름은 "로컬 호스트"여야 하며 이러한 값을 변경하지 않는 한 포트는 "27017"이어야 합니다. 원격 시스템에서 Planetmint 노드를 실행 중이며 해당 시스템에 SSH할 수 있는 경우에도 마찬가지입니다. + +원격 시스템에서 Planetmint 노드를 실행하고 MongoDB를 auth를 사용하고 공개적으로 액세스할 수 있도록 구성한 경우(권한이 있는 사용자에게) 호스트 이름과 포트를 확인할 수 있습니다. + +쿼리하기 +------------ + +Planetmint 노드 운영자는 로컬 MongoDB 인스턴스에 대한 전체 액세스 권한을 가지므로 실행하는데 MongoDB의 다음의 API를 사용할 수 있습니다: + +- [the Mongo Shell](https://docs.mongodb.com/manual/mongo/) +- [MongoDB Compass](https://www.mongodb.com/products/compass) +- one of [the MongoDB drivers](https://docs.mongodb.com/ecosystem/drivers/), such as [PyMongo](https://api.mongodb.com/python/current/), or +- MongoDB 쿼리에 대한 서드파티툴, RazorSQL, Studio 3T, Mongo Management Studio, NoSQLBooster for MongoDB, or Dr. Mongo. + +Note + +SQL을 이용해 mongoDB 데이터베이스를 쿼리할 수 있습니다. 예를 들어: + + * Studio 3T: "[How to Query MongoDB with SQL](https://studio3t.com/whats-new/how-to-query-mongodb-with-sql/)" + * NoSQLBooster for MongoDB: "[How to Query MongoDB with SQL SELECT](https://mongobooster.com/blog/query-mongodb-with-sql/)" + +예를 들어, 기본 Planetmint 노드를 실행하는 시스템에 있는 경우 Mongo Shell (``mongo``)을 사용하여 연결하고 다음과 같이 볼 수 있습니다. + + $ mongo + MongoDB shell version v3.6.5 + connecting to: mongodb://127.0.0.1:27017 + MongoDB server version: 3.6.4 + ... + > show dbs + admin 0.000GB + bigchain 0.000GB + config 0.000GB + local 0.000GB + > use bigchain + switched to db bigchain + > show collections + abci_chains + assets + blocks + elections + metadata + pre_commit + transactions + utxos + validators + +위 예제는 몇 가지 상황을 보여줍니다: + +- 호스트 이름이나 포트를 지정하지 않으면 Mongo Shell은 각각 `localhost`와 `27017`으로 가정합니다. (`localhost`는 우분투에 IP주소를 127.0.0.1로 설정했습니다.) + + +* BigchainDB는 데이터를 `bigchain`이라는 데이터베이스에 저장합니다. +* `bigchain` 데이터베이스에는 여러 [collections](https://docs.mongodb.com/manual/core/databases-and-collections/)가 포함되어 있습니다. +* 어떤 컬렉션에도 투표가 저장되지 않습니다. 이런 데이터는 모두 자체(LevelDB) 데이터베이스에 의해 처리되고 저장됩니다. + +컬렉션에 대한 예시 문서 +--------------------------------------- + +``bigchain`` 데이터베이스의 가장 흥미로운 부분은 아래와 같습니다: + +- transactions +- assets +- metadata +- blocks + +`db.assets.findOne()` 은 MongoDB 쿼리를 사용하여 이러한 컬렉션들을 탐색할 수 있습니다. + +### 트랜잭션에 대한 예시 문서 + +transaction 컬렉션에서 CREATE 트랜잭션에는 추가 `"_id"` 필드(MongoDB에 추가됨)가 포함되며 `"asset"`과 `"metadata"` 필드에는 데이터가 저장되어 있지 않습니다. + + { + "_id":ObjectId("5b17b9fa6ce88300067b6804"), + "inputs":[…], + "outputs":[…], + "operation":"CREATE", + "version":"2.0", + "id":"816c4dd7…851af1629" + } + +A TRANSFER transaction from the transactions collection is similar, but it keeps its `"asset"` field. + + { + "_id":ObjectId("5b17b9fa6ce88300067b6807"), + "inputs":[…], + "outputs":[…], + "operation":"TRANSFER", + "asset":{ + "id":"816c4dd7ae…51af1629" + }, + "version":"2.0", + "id":"985ee697d…a3296b9" + } + +### assets에 대한 예시 문서 + +assets에 대한 기술에는 MongoDB가 추가한 `"_id"` 분야와 CREATE 거래에서 나온 `asset.data` 그리고 `"id"` 세 가지 최상위 분야로 구성되어 있습니다. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +{ + "_id":ObjectId("5b17b9fe6ce88300067b6823"), + "data":{ + "type":"cow", + "name":"Mildred" + }, + "id":"96002ef8740…45869959d8" +} + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +### metadata에 대한 예시 문서 + + +metadata 컬렉션의 문서는 MongoDB가 추가한 `"_id"`필드와 거래에서 나온 `asset.data`그리고 거래에서 나온 ``"id"`` 세 가지 최상위 분야로 구성되어 있습니다. + + { + "_id":ObjectId("5b17ba006ce88300067b683d"), + "metadata":{ + "transfer_time":1058568256 + }, + "id":"53cba620e…ae9fdee0" + } + +### blocks에 대한 예시 문서 + + { + "_id":ObjectId("5b212c1ceaaa420006f41c57"), + "app_hash":"2b0b75c2c2…7fb2652ce26c6", + "height":17, + "transactions":[ + "5f1f2d6b…ed98c1e" + ] + } + +## 노드 operator가 외부 유저에게 보낼 수 있는 것 + +각 노드 operator는 외부 사용자가 자신의 로컬 MongoDB 데이터베이스에서 정보를 얻는 방법을 결정할 수 있습니다. 그들은 다음과 같은 것들을 보낼 수 있습니다: + +- 외부유저를 쿼리 처리하는 로컬 MongoDB 데이터베이스 한된 제한된 권한을 가진 역할을 가진 MongoDB 사용자 예) read-only +- 제한된 미리 정의된 쿼리 집합을 허용하는 제한된 HTTP API, [Planetmint 서버에서 제공하는 HTTP API](http://bigchaindb.com/http-api), 혹은Django, Express, Ruby on Rails, or ASP.NET.를 이용해 구현된 커스텀 HTTP API +- 다른 API(예: GraphQL API) 제3자의 사용자 정의 코드 또는 코드를 사용하여 수행할 수 있습니다.. + +각 노드 operator는 로컬 MongoDB 데이터베이스에 대한 다른 레벨 또는 유형의 액세스를 노출할 수 있습니다. +예를 들어, 한 노드 operator가 최적화된 [공간 쿼리](https://docs.mongodb.com/manual/reference/operator/query-geospatial/)를 전문으로 제공하기로 정할 수 있습니다. + +보안 고려사항 +----------------------- + +Planetmint 버전 1.3.0 이전 버전에서는 하나의 MongoDB 논리 데이터베이스가 있었기 때문에 외부 사용자에게 데이터베이스를 노출하는 것은 매우 위험했으며 권장되지 않습니다. "Drop database"는 공유된 MongoDB 데이터베이스를 삭제합니다. + +Planetmint 버전 2.0.0 이상에선 각 노드에 고유한 독립 로컬 MongoDB 데이터베이스가 존재합니다. 노드 간 통신은 아래 그림 1에서와 같이 MongoDB 프로토콜이 아닌 Tendermint 프로토콜을 사용하여 수행됩니다. 노드의 로컬 MongoDB 데이터베이스가 손상되어도 다른 노드는 영향을 받지 않습니다. + +![image](https://user-images.githubusercontent.com/36066656/48752907-f1dcd600-ecce-11e8-95f4-3cdeaa1dc4c6.png) + +Figure 1: A Four-Node Planetmint 2.0 Network + +퍼포먼스 및 요금 고려사항 +----------------------------------- + +쿼리 프로세싱은 상당히 많은 리소스를 소모할 수 있으므로, Planetmint 서버 및 Tendermint Core와 별도의 컴퓨터에서 MongoDB를 실행하는 것이 좋습니다. + +노드 operator 는 조회에 사용되는 리소스를 측정하여 조회를 요청한 사람은 누구든지 요금을 청구할 수 있습니다. + +일부 쿼리는 너무 오래 걸리거나 리소스를 너무 많이 사용할 수 있습니다. 노드 operator는 사용할 수 있는 리소스에 상한을 두고, 초과된다면 중지(또는 차단)해야 합니다. + +MongoDB 쿼리를 더욱 효율적으로 만들기 위해 [인덱스](https://docs.mongodb.com/manual/indexes/)를 만들 수 있습니다. 이러한 인덱스는 노드 operator 또는 일부 외부 사용자가 생성할 수 있습니다(노드 운영자가 허용하는 경우). 인덱스는 비어 있지 않습니다. 새 데이터를 컬렉션에 추가할 때마다 해당 인덱스를 업데이트해야 합니다. 노드 운영자는 이러한 요금을 인덱스를 생성한 사람에게 전달하고자 할 수 있습니다. mongoDB에서는 [단일 컬렉션은 64개 이하의 인덱스를 가질 수 있습니다](https://docs.mongodb.com/manual/reference/limits/#Number-of-Indexes-per-Collection). + +Tendermint voting 파워가 0인 노드인 추종자 노드를 생성할 수 있다. 여전히 모든 데이터의 복사본이 있으므로 읽기 전용 노드로 사용할 수 있습니다. Follower 노드는 투표 검증자의 작업 부하에 영향을 미치지 않고 서비스로 전문화된 쿼리를 제공할 수 있습니다(쓰기도 가능). 팔로워의 팔로워들도 있을 수 있습니다. + +자바스크립트 쿼리 코드 예시 +------------------------------ + +[MongoDB node.js 드라이버](https://mongodb.github.io/node-mongodb-native/?jmp=docs)와 같은 MongoDB 드라이버를 사용하여 다음 중 하나를 사용하여 노드의 MongoDB 데이터베이스에 연결할 수 있습니다. 여기 자바스크립트 쿼리 코드에 대한 링크가 있습니다. + +- [The Planetmint JavaScript/Node.js driver source code](https://github.com/bigchaindb/js-bigchaindb-driver) +- [Example code by @manolodewiner](https://github.com/manolodewiner/query-mongodb-bigchaindb/blob/master/queryMongo.js) +- [More example code by @manolodewiner](https://github.com/bigchaindb/bigchaindb/issues/2315#issuecomment-392724279) \ No newline at end of file diff --git a/docs/root/source/korean/smart-contracts_ko.md b/docs/root/source/korean/smart-contracts_ko.md new file mode 100644 index 0000000..7df6dc4 --- /dev/null +++ b/docs/root/source/korean/smart-contracts_ko.md @@ -0,0 +1,17 @@ + + +Planetmint 및 스마트계약 +============================== + +BigchainDB에는 스마트 계약 (즉, 컴퓨터 프로그램)의 소스 코드를 저장할 수 있지만 BigchainDB는 임의의 스마트 계약을 실행하지 않습니다. + +BigchainDB는 대체 가능한 자산과 대체 할 수없는 자산 모두를 전송할 수있는 권한을 가진 사람을 시행하는 데 사용할 수 있습니다. 이중 지출을 막을 것입니다. 즉, ERC-20 (대체 가능한 토큰) 또는 ERC-721 (대체 할 수없는 토큰) 스마트 계약 대신 Planetmint 네트워크를 사용할 수 있습니다. + +자산 이전 권한은 쓰기 권한으로 해석 될 수 있으므로 로그, 저널 또는 감사 내역에 기록 할 수있는 사람을 제어하는데 사용할 수 있습니다. [BigchainDB의 사용 권한](https://github.com/bigchaindb/bigchaindb/blob/master/docs/root/source/korean/permissions-ko.md)에 대한 자세한 내용은 페이지에서 확인하십시오. + +Planetmint 네트워크는 oracles 또는 체인 간 통신 프로토콜을 통해 다른 블록 체인 네트워크에 연결할 수 있습니다. 이는 BigchainDB를 다른 블록 체인을 사용하여 임의의 스마트 계약을 실행하는 솔루션의 일부로 사용할 수 있음을 의미합니다. diff --git a/docs/root/source/korean/store-files_ko.md b/docs/root/source/korean/store-files_ko.md new file mode 100644 index 0000000..fe6b50d --- /dev/null +++ b/docs/root/source/korean/store-files_ko.md @@ -0,0 +1,14 @@ + + +# BigchainDB에 파일을 저장하는 방법 + +Planetmint 네트워크에 파일을 저장할 수는 있지만 그렇게하지 않는 것이 좋습니다. 파일이 아닌 구조화 된 데이터를 저장, 인덱싱 및 쿼리하는 데 가장 적합합니다. + +분산 된 파일 저장소를 원하면 Storj, Sia, Swarm 또는 IPFS / Filecoin을 확인하십시오. 파일 URL, 해시 또는 기타 메타 데이터를 Planetmint 네트워크에 저장할 수 있습니다. + +Planetmint 네트워크에 파일을 저장해야하는 경우,이를 수행하는 한 가지 방법은 긴 Base64 문자열로 변환 한 다음 해당 문자열을 하나 이상의 Planetmint 트랜잭션 (CREATE 트랜잭션의 `asset.data`)에 저장하는 것입니다 , 또는 어떤 거래의 `메타데이터` 일 수도있다. diff --git a/docs/root/source/korean/terminology_kor.md b/docs/root/source/korean/terminology_kor.md new file mode 100644 index 0000000..47f398d --- /dev/null +++ b/docs/root/source/korean/terminology_kor.md @@ -0,0 +1,26 @@ + + +# 용어 + +BigchainDB와 관련돈 몇 가지 전문화된 용어가 있습니다. 시작하기에 앞서, 최소한 다음과 같은 사항을 알아야합니다. + +## Planetmint 노드 + +**Planetmint 노드**는 [Planetmint 서버](https://docs.bigchaindb.com/projects/server/en/latest/introduction.html) 및 관련된 소프트웨어를 실행하는 시스템(또는 논리적인 시스템)입니다. 각각의 노드는 한 개인이나 조직에 의해 제어될 수 있습니다. + +## Planetmint 네트워크 + +Planetmint 노드들의 집합은 서로 연결하여 **Planetmint 네트워크**를 형성할 수 있습니다. 해당 네트워크에서 각각의 노드는 동일한 소프트웨어를 실행합니다. Planetmint 네트워크는 모니터링 같은 것들을 하기 위한 추가적인 시스템이 있을 수 있습니다. + +## Planetmint 컨소시엄 + +Planetmint 네트워크에 노드들을 실행하는 사람과 조직은 **Planetmint 컨소시엄**(즉, 다른 조직)에 속합니다. 컨소시엄은 결정을 하기 위해 일종의 거버넌스 구조를 가져야합니다. 만약 Planetmint 네트워크가 단 하나의 회사에 의해서 운영된다면, "컨소시엄"은 단지 그 회사일 뿐입니다. + +**Planetmint 네트워크와 컨소시엄의 차이는 무엇일까요?** + +Planetmint 네트워크는 단지 연결된 노드들의 집합입니다. 컨소시엄은 하나의 Planetmint 네트워크를 가지는 조직이며, 해당 네트워크에서 각각의 노드는 다른 운영자를 가집니다. diff --git a/docs/root/source/korean/transaction-concepts_ko.md b/docs/root/source/korean/transaction-concepts_ko.md new file mode 100644 index 0000000..011256b --- /dev/null +++ b/docs/root/source/korean/transaction-concepts_ko.md @@ -0,0 +1,61 @@ + + +# 트랜잭션 개념 + +*트랜잭션*은 물건 (예 : 자산)을 등록, 발행, 생성 또는 전송하는 데 사용됩니다. + +트랜잭션은 BigchainDB가 저장하는 가장 기본적인 종류의 레코드입니다. CREATE 트랜잭션과 TRANSFER 트랜잭션의 두 종류가 있습니다. + + +## 트랜잭션 생성 + +CREATE 트랜잭션은 BigchainDB에서 한 가지 (또는 자산)의 이력을 등록, 발행, 생성 또는 다른 방법으로 시작하는 데 사용될 수 있습니다. 예를 들어, 신원이나 창작물을 등록 할 수 있습니다. 이러한 것들을 종종 "자산"이라고 부르지만 literal 자산이 아닐 수도 있습니다. + +BigchainDB는 Planetmint Server v0.8.0부터 나눌 수있는 자산을 지원합니다. 이는 "공유"의 초기 숫자로 자산을 생성 / 등록 할 수 있음을 의미합니다. 예를 들어, CREATE 트랜잭션은 50 개의 오크 나무로 된 트럭로드를 등록 할 수 있습니다. 분할 가능한 자산의 각 주식은 서로 공유 할 수 있어야합니다. 주식은 대체 가능해야합니다. + +CREATE 트랜잭션은 하나 이상의 출력을 가질 수 있습니다. 각 출력에는 관련 금액이 있습니다. 출력에 연결된 공유 수입니다. 예를 들어 자산이 50 개의 오크 나무로 구성되어있는 경우 한 출력에는 한 소유자 세트에 35 개의 오크 나무가 있고 다른 출력에는 다른 소유자 세트에는 15 개의 오크 나무가있을 수 있습니다. + +또한 각 출력에는 연관된 조건이 있습니다. 출력을 전송 / 소비하기 위해 충족되어야하는 조건 (TRANSFER 트랜잭션에 의해). BigchainDB는 다양한 조건을 지원합니다. 자세한 내용은 관련 [Planetmint 트랜잭션 Spec](https://github.com/bigchaindb/BEPs/tree/master/tx-specs/)과 관련된 **트랜잭션 구성 요소 : 조건 섹션**을 참조하십시오. + +![Example Planetmint CREATE transaction](./_static/CREATE_example.png) + +위의 예제에서는 Planetmint CREATE 트랜잭션 다이어그램을 보여줍니다. Pam은 자산 3 주를 소유 / 통제하고 다른 주식은 없습니다 (다른 산출물이 없으므로). + +각 출력에는 해당 출력의 조건과 연관된 모든 공개 키 목록이 있습니다. 다시 말하면, 그 목록은 "소유자"의 목록으로 해석 될 수 있습니다.보다 정확한 단어는 이행자, 서명자, 컨트롤러 또는 이전 가능 요소 일 수 있습니다. 관련 [Planetmint Transactions Spec](https://github.com/bigchaindb/BEPs/tree/master/tx-specs/) **소유자에 관한 참고 사항** 섹션을 참조하십시오. + +CREATE 트랜잭션은 모든 소유자가 서명해야합니다. (만약 당신이 그 서명을 원한다면, 그것은 인코딩되었지만 하나의 입력의 "이행"에있다.) + +## 트랜잭션 이전 + +트랜잭션 이전은 다른 트랜잭션 (CREATE 트랜잭션 또는 다른 TRANSFER 트랜잭션)에서 하나 이상의 출력을 전송 / 소비 할 수 있습니다. 이러한 출력물은 모두 동일한 자산과 연결되어야합니다. TRANSFER 트랜잭션은 한 번에 하나의 자산의 공유 만 전송할 수 있습니다. + +트랜잭션 이전의 각 입력은 다른 트랜잭션의 한 출력에 연결됩니다. 각 입력은 전송 / 소비하려는 출력의 조건을 충족해야합니다. + +트랜잭션 이전은 위에서 설명한 CREATE 트랜잭션과 마찬가지로 하나 이상의 출력을 가질 수 있습니다. 투입물에 들어오는 총 주식 수는 산출물에서 나가는 총 주식 수와 같아야합니다. + +![Example Planetmint transactions](./_static/CREATE_and_TRANSFER_example.png) + +위 그림은 두 개의 Planetmint 트랜잭션, CREATE 트랜잭션 및 TRANSFER 트랜잭션의 다이어그램을 보여줍니다. CREATE 트랜잭션은 이전 다이어그램과 동일합니다. TRANSFER 트랜잭션은 Pam의 출력을 소비하므로 TRANSFER 트랜잭션의 입력에는 Pam의 유효한 서명 (즉, 유효한 이행)이 포함되어야합니다. TRANSFER 트랜잭션에는 두 개의 출력이 있습니다. Jim은 하나의 공유를 가져오고 Pam은 나머지 두 개의 공유를 가져옵니다. + +용어 : "Pam, 3"출력을 "소비 된 트랜잭션 출력"이라고하며 "Jim, 1"및 "Pam, 2"출력을 "사용되지 않은 트랜잭션 출력"(UTXO)이라고합니다. + +**예제 1:** 빨간 차가 Joe가 소유하고 관리한다고 가정합니다. 자동차의 현재 전송 조건에서 Joe가 유효한 전송을 서명해야한다고 가정합니다. Joe는 Joe의 서명 (현재 출력 조건을 충족시키기 위해)과 Rae가 유효한 전송을 서명해야한다는 새로운 출력 조건을 포함하는 입력을 포함하는 TRANSFER 트랜잭션을 작성할 수 있습니다. + +**예제 2:** 예를 들어 동일한 자산 유형의 이전에 전송되지 않은 4 개의 자산에 대한 출력 조건을 충족하는 TRANSFER 트랜잭션을 생성 할 수 있습니다. 종이 클립. 총 금액은 20, 10, 45 및 25 일 수 있으며, 말하자면 총 100 개의 클립입니다. 또한 TRANSFER 트랜잭션은 새로운 전송 조건을 설정합니다. 예를 들어, Gertrude가 서명하는 경우에만 60 개의 클립 클립이 전송 될 수 있으며 Jack과 Kelly가 서명하는 경우에만 40 개의 클립 클립이 전송 될 수 있습니다. 들어오는 클립 클립의 합계가 나가는 클립 클립의 합계와 같아야합니다 (100). + +## 트랜잭션 유효성 + +언제 트랜잭션이 유효한지 유효성을 검사하는 것에 관해 해당 블로그에 게시되어있습니다. *The Planetmint Blog*: +["What is a Valid Transaction in Planetmint?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598) (Note: That post was about Planetmint Server v1.0.0.) + +Each [Planetmint Transactions Spec](https://github.com/bigchaindb/BEPs/tree/master/tx-specs/) documents the conditions for a transaction (of that version) to be valid. + +## 트랜잭션 예시 + +아래의 [HTTP API 문서](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html)와 [the Python 드라이버 문서](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html)에는 예제 Planetmint 트랜잭션이 있습니다. +. diff --git a/docs/root/source/properties.md b/docs/root/source/properties.md new file mode 100644 index 0000000..72c8c64 --- /dev/null +++ b/docs/root/source/properties.md @@ -0,0 +1,60 @@ + + +# Properties of Planetmint + +### Decentralization + +Decentralization means that no one owns or controls everything, and there is no single point of failure. + +Ideally, each node in a Planetmint network is owned and controlled by a different person or organization. Even if the network lives within one organization, it's still preferable to have each node controlled by a different person or subdivision. + +We use the phrase "Planetmint consortium" (or just "consortium") to refer to the set of people and/or organizations who run the nodes of a Planetmint network. A consortium requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each consortium, but it can be very decentralized. + +A consortium can increase its decentralization (and its resilience) by increasing its jurisdictional diversity, geographic diversity, and other kinds of diversity. + +There’s no node that has a long-term special position in the Planetmint network. All nodes run the same software and perform the same duties. + +If someone has (or gets) admin access to a node, they can mess with that node (e.g. change or delete data stored on that node), but those changes should remain isolated to that node. The Planetmint network can only be compromised if more than one third of the nodes get compromised. See the [Tendermint documentation](https://tendermint.com/docs/introduction/introduction.html) for more details. + +It’s worth noting that not even the admin or superuser of a node can transfer assets. The only way to create a valid transfer transaction is to fulfill the current crypto-conditions on the asset, and the admin/superuser can’t do that because the admin user doesn’t have the necessary information (e.g. private keys). + +### Byzantine Fault Tolerance + +[Tendermint](https://tendermint.com/) is used for consensus and transaction replication, +and Tendermint is [Byzantine Fault Tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance). + +### Node Diversity + +Steps should be taken to make it difficult for any one actor or event to control or damage “enough” of the nodes. (Because Planetmint Server uses Tendermint, "enough" is ⅓.) There are many kinds of diversity to consider, listed below. It may be quite difficult to have high diversity of all kinds. + +1. **Jurisdictional diversity.** The nodes should be controlled by entities within multiple legal jurisdictions, so that it becomes difficult to use legal means to compel enough of them to do something. +1. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems. +1. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes. +1. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a consortium. For example, it provides the consortium with a source of various ideas for addressing challenges. + +Note: If all the nodes are running the same code, i.e. the same implementation of Planetmint, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of Planetmint Server (e.g. one in Python, one in Go, etc.), so that a consortium could also have a diversity of server implementations. Similar remarks can be made about the operating system. + +### Immutability + +The blockchain community often describes blockchains as “immutable.” If we interpret that word literally, it means that blockchain data is unchangeable or permanent, which is absurd. The data _can_ be changed. For example, a plague might drive humanity extinct; the data would then get corrupted over time due to water damage, thermal noise, and the general increase of entropy. + +It’s true that blockchain data is more difficult to change (or delete) than usual. It's more than just "tamper-resistant" (which implies intent), blockchain data also resists random changes that can happen without any intent, such as data corruption on a hard drive. Therefore, in the context of blockchains, we interpret the word “immutable” to mean *practically* immutable, for all intents and purposes. (Linguists would say that the word “immutable” is a _term of art_ in the blockchain community.) + +Blockchain data can be made immutable in several ways: + +1. **No APIs for changing or deleting data.** Blockchain software usually doesn't expose any APIs for changing or deleting the data stored in the blockchain. Planetmint has no such APIs. This doesn't prevent changes or deletions from happening in _other_ ways; it's just one line of defense. +1. **Replication.** All data is replicated (copied) to several different places. The higher the replication factor, the more difficult it becomes to change or delete all replicas. +1. **Internal watchdogs.** All nodes monitor all changes and if some unallowed change happens, then appropriate action can be taken. +1. **External watchdogs.** A consortium may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For a consortium with publicly-readable data, the public can act as an auditor. +1. **Economic incentives.** Some blockchain systems make it very expensive to change old stored data. Examples include proof-of-work and proof-of-stake systems. Planetmint doesn't use explicit incentives like those. +1. Data can be stored using fancy techniques, such as error-correction codes, to make some kinds of changes easier to undo. +1. **Cryptographic signatures** are often used as a way to check if messages (e.g. transactions) have been tampered with enroute, and as a way to verify who signed the messages. In Planetmint, each transaction must be signed by one or more parties. +1. **Full or partial backups** may be recorded from time to time, possibly on magnetic tape storage, other blockchains, printouts, etc. +1. **Strong security.** Node owners can adopt and enforce strong security policies. + + diff --git a/docs/root/source/query.rst b/docs/root/source/query.rst new file mode 100644 index 0000000..fda0805 --- /dev/null +++ b/docs/root/source/query.rst @@ -0,0 +1,229 @@ + +.. Copyright © 2020 Interplanetary Database Association e.V., + Planetmint and IPDB software contributors. + SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) + Code is Apache-2.0 and docs are CC-BY-4.0 + +Queries in Planetmint +===================== + +A node operator can use the full power of MongoDB's query engine to search and query all stored data, including all transactions, assets and metadata. +The node operator can decide for themselves how much of that query power they expose to external users. + +Blog Post with Example Queries +------------------------------ + +We wrote a blog post in The Planetmint Blog to show +how to use some MongoDB tools to query a Planetmint node's MongoDB database. +It includes some specific example queries for data +about custom cars and their ownership histories. +`Check it out `_. + +How to Connect to MongoDB +------------------------- + +Before you can query a MongoDB database, you must connect to it, and to do that, you need to know its hostname and port. + +If you're running a Planetmint node on your local machine (e.g. for dev and test), then the hostname should be ``localhost`` and the port should be ``27017``, unless you did something to change those values. If you're running a Planetmint node on a remote machine and you can SSH to that machine, then the same is true. + +If you're running a Planetmint node on a remote machine and you configured its MongoDB to use auth and to be publicly-accessible (to people with authorization), then you can probably figure out its hostname and port. + +How to Query +------------ + +A Planetmint node operator has full access to their local MongoDB instance, so they can use any of MongoDB's APIs for running queries, including: + +- `the Mongo Shell `_, +- `MongoDB Compass `_, +- one of `the MongoDB drivers `_, such as `PyMongo `_, or +- a third-party tool for doing MongoDB queries, such as RazorSQL, Studio 3T, Mongo Management Studio, NoSQLBooster for MongoDB, or Dr. Mongo. + +.. note:: + + It's possible to do query a MongoDB database using SQL. For example: + + * Studio 3T: "`How to Query MongoDB with SQL `_" + * NoSQLBooster for MongoDB: "`How to Query MongoDB with SQL SELECT `_" + +For example, if you're on a machine that's running a default Planetmint node, then you can connect to it using the Mongo Shell (``mongo``) and look around like so: + +.. code:: + + $ mongo + MongoDB shell version v3.6.5 + connecting to: mongodb://127.0.0.1:27017 + MongoDB server version: 3.6.4 + ... + > show dbs + admin 0.000GB + bigchain 0.000GB + config 0.000GB + local 0.000GB + > use bigchain + switched to db bigchain + > show collections + abci_chains + assets + blocks + elections + metadata + pre_commit + transactions + utxos + validators + +The above example illustrates several things: + +* When you don't specify the hostname or port, the Mongo Shell assumes they are ``localhost`` and ``27017``, respectively. (``localhost`` had IP address 127.0.0.1 on the machine in question, an Ubuntu machine.) +* Planetmint stores its data in a database named ``bigchain``. +* The ``bigchain`` database contains several `collections `_. +* Votes aren't stored in any collection, currently. They are all handled and stored by Tendermint in its own (LevelDB) database. + +Example Documents from Some Collections +--------------------------------------- + +The most interesting collections in the ``bigchain`` database are: + +- transactions +- assets +- metadata +- blocks + +You can explore those collections using MongoDB queries such as ``db.assets.findOne()``. We now show some example documents from each of those collections. + +Example Documents from transactions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A CREATE transaction from the transactions collection includes an extra ``"_id"`` field (added by MongoDB) and is missing its ``"asset"`` and ``"metadata"`` fields: that data was removed and stored in the assets and metadata collections. + +.. code:: + + { + "_id":ObjectId("5b17b9fa6ce88300067b6804"), + "inputs":[…], + "outputs":[…], + "operation":"CREATE", + "version":"2.0", + "id":"816c4dd7…851af1629" + } + +A TRANSFER transaction from the transactions collection is similar, but it keeps its ``"asset"`` field. + +.. code:: + + { + "_id":ObjectId("5b17b9fa6ce88300067b6807"), + "inputs":[…], + "outputs":[…], + "operation":"TRANSFER", + "asset":{ + "id":"816c4dd7ae…51af1629" + }, + "version":"2.0", + "id":"985ee697d…a3296b9" + } + +Example Document from assets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A document from the assets collection has three top-level fields: an ``"_id"`` field added by MongoDB, the ``asset.data`` from a CREATE transaction, and the ``"id"`` of the CREATE transaction it came from. + +.. code:: + + { + "_id":ObjectId("5b17b9fe6ce88300067b6823"), + "data":{ + "type":"cow", + "name":"Mildred" + }, + "id":"96002ef8740…45869959d8" + } + +Example Document from metadata +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A document from the metadata collection has three top-level fields: an ``"_id"`` field added by MongoDB, the ``metadata`` from a transaction, and the ``"id"`` of the transaction it came from. + +.. code:: + + { + "_id":ObjectId("5b17ba006ce88300067b683d"), + "metadata":{ + "transfer_time":1058568256 + }, + "id":"53cba620e…ae9fdee0" + } + +Example Document from blocks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: + + { + "_id":ObjectId("5b212c1ceaaa420006f41c57"), + "app_hash":"2b0b75c2c2…7fb2652ce26c6", + "height":17, + "transactions":[ + "5f1f2d6b…ed98c1e" + ] + } + +What a Node Operator Can Expose to External Users +------------------------------------------------- + +Each node operator can decide how they let external users get information from their local MongoDB database. They could expose: + +- their local MonogoDB database itself to queries from external users, maybe as a MongoDB user with a role that has limited privileges, e.g. read-only. +- a limited HTTP API, allowing a restricted set of predefined queries, such as `the HTTP API provided by Planetmint Server `_, or a custom HTTP API implemented using Django, Express, Ruby on Rails, or ASP.NET. +- some other API, such as a GraphQL API. They could do that using custom code or code from a third party. + +Each node operator can expose a different level or type of access to their local MongoDB database. +For example, one node operator might decide to specialize in offering optimized `geospatial queries `_. + +Security Considerations +----------------------- + +In Planetmint version 1.3.0 and earlier, there was one logical MongoDB database, so exposing that database to external users was very risky, and was not recommended. +"Drop database" would delete that one shared MongoDB database. + +In Planetmint version 2.0.0 and later, each node has its own isolated local MongoDB database. +Inter-node communications are done using Tendermint protocols, not MongoDB protocols, as illustrated in Figure 1 below. +If a node's local MongoDB database gets compromised, none of the other MongoDB databases (in the other nodes) will be affected. + +.. figure:: _static/schemaDB.png + :alt: Diagram of a four-node Planetmint 2.0 network + :align: center + + Figure 1: A Four-Node Planetmint 2.0 Network + +.. raw:: html + +
+
+
+ +Performance and Cost Considerations +----------------------------------- + +Query processing can be quite resource-intensive, so it's a good idea to have MongoDB running in a separate machine from those running Planetmint Server and Tendermint Core. + +A node operator might want to measure the resources used by a query, so they can charge whoever requested the query accordingly. + +Some queries can take too long or use too many resources. A node operator should put upper bounds on the resources that a query can use, and halt (or prevent) any query that goes over. + +To make MongoDB queries more efficient, one can create `indexes `_. Those indexes might be created by the node operator or by some external users (if the node operator allows that). It's worth noting that indexes aren't free: whenever new data is appended to a collection, the corresponding indexes must be updated. The node operator might want to pass those costs on to whoever created the index. Moreover, in MongoDB, `a single collection can have no more than 64 indexes `_. + +One can create a follower node: a node with Tendermint voting power 0. It would still have a copy of all the data, so it could be used as read-only node. A follower node could offer specialized queries as a service without affecting the workload on the voting validators (which can also write). There could even be followers of followers. + +JavaScript Query Code Examples +------------------------------ + +One can connect to a node's MongoDB database using any +of the MongoDB drivers, such as `the MongoDB Node.js driver +`_. +Here are some links to example JavaScript code that queries a +Planetmint node's MongoDB database: + +- `The Planetmint JavaScript/Node.js driver source code `_ +- `Example code by @manolodewiner `_ +- `More example code by @manolodewiner `_ diff --git a/docs/root/source/terminology.md b/docs/root/source/terminology.md new file mode 100644 index 0000000..3b8ad90 --- /dev/null +++ b/docs/root/source/terminology.md @@ -0,0 +1,81 @@ + + +# Terminology + +There is some specialized terminology associated with Planetmint. To get started, you should at least know the following: + +### Planetmint Node + +A **Planetmint node** is a machine (or logical machine) running [Planetmint Server](https://docs.bigchaindb.com/projects/server/en/latest/introduction.html) and related software. Each node is controlled by one person or organization. + +### Planetmint Network + +A set of Planetmint nodes can connect to each other to form a **Planetmint network**. Each node in the network runs the same software. A Planetmint network may have additional machines to do things such as monitoring. + +### Planetmint Consortium + +The people and organizations that run the nodes in a Planetmint network belong to a **Planetmint consortium** (i.e. another organization). A consortium must have some sort of governance structure to make decisions. If a Planetmint network is run by a single company, then the "consortium" is just that company. + +**What's the Difference Between a Planetmint Network and a Consortium?** + +A BigchaindB network is just a bunch of connected nodes. A consortium is an organization which has a Planetmint network, and where each node in that network has a different operator. + +### Transactions + +Are described in detail in `Planetmint Transactions Spec `_ . + +## Permissions in Planetmint + +Planetmint lets users control what other users can do, to some extent. That ability resembles "permissions" in the \*nix world, "privileges" in the SQL world, and "access control" in the security world. + + +### Permission to Spend/Transfer an Output + +In Planetmint, every output has an associated condition (crypto-condition). + +To spend/transfer an unspent output, a user (or group of users) must fulfill the condition. Another way to say that is that only certain users have permission to spend the output. The simplest condition is of the form, "Only someone with the private key corresponding to this public key can spend this output." Much more elaborate conditions are possible, e.g. "To spend this output, …" + +- "…anyone in the Accounting Group can sign." +- "…three of these four people must sign." +- "…either Bob must sign, or both Tom and Sylvia must sign." + +Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, Planetmint doesn't permit anyone to "double spend" an output. + + +### Write Permissions + +When someone builds a TRANSFER transaction, they can put an arbitrary JSON object in the ``metadata`` field (within reason; real Planetmint networks put a limit on the size of transactions). That is, they can write just about anything they want in a TRANSFER transaction. + +Does that mean there are no "write permissions" in Planetmint? Not at all! + +A TRANSFER transaction will only be valid (allowed) if its inputs fulfill some previous outputs. The conditions on those outputs will control who can build valid TRANSFER transactions. In other words, one can interpret the condition on an output as giving "write permissions" to certain users to write something into the history of the associated asset. + +As a concrete example, you could use Planetmint to write a public journal where only you have write permissions. Here's how: First you'd build a CREATE transaction with the ``asset.data`` being something like ``{"title": "The Journal of John Doe"}``, with one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. +Each time you want to append something to your journal, you'd build a new TRANSFER transaction with your latest entry in the ``metadata`` field, e.g. + +.. code-block:: json + + {"timestamp": "1508319582", + "entry": "I visited Marmot Lake with Jane."} + +The TRANSFER transaction would have one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. And so on. Only you would be able to append to the history of that asset (your journal). + +The same technique could be used for scientific notebooks, supply-chain records, government meeting minutes, and so on. + +You could do more elaborate things too. As one example, each time someone writes a TRANSFER transaction, they give *someone else* permission to spend it, setting up a sort of writers-relay or chain letter. + +.. note:: + + Anyone can write any JSON (again, within reason) in the ``asset.data`` field of a CREATE transaction. They don't need permission. + + +### Role-Based Access Control (RBAC) + +In September 2017, we published a [blog post about how one can define an RBAC sub-system on top of Planetmint](https://blog.bigchaindb.com/role-based-access-control-for-bigchaindb-assets-b7cada491997). +At the time of writing (January 2018), doing so required the use of a plugin, so it's not possible using standard Planetmint (which is what's available on the [IPDB Testnet](https://test.ipdb.io/>). That may change in the future. +If you're interested, `contact IPDB `_. diff --git a/docs/upgrade-guides/v0.10-v1.0.md b/docs/upgrade-guides/v0.10-v1.0.md new file mode 100644 index 0000000..db533a9 --- /dev/null +++ b/docs/upgrade-guides/v0.10-v1.0.md @@ -0,0 +1,439 @@ + + +# Updating from Planetmint v0.10 to v1.0 + +Planetmint v1.0 stands for backwards compatibility. This means that all +following [minor](http://semver.org/) releases after version 1.0 will always be +backwards-compatible to previous versions. + +For all future releases, we commit to not introduce breaking changes to the +public interfaces of Planetmint's: + +- [Data + model](https://docs.bigchaindb.com/projects/server/en/latest/data-models/index.html) +- [HTTP + API](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html) +- [WebSocket Event Stream + API](https://docs.bigchaindb.com/projects/server/en/latest/websocket-event-stream-api.html) + + +As we saw the version bump to v1.0 as our last chance in a while to fix minor +annoyances, we intentionally did clean up on the above interfaces. In this +document, we'd like to give a comprehensive summary of those changes to allow +you to upgrade efficiently. + +The next sections will go over each of the above mentioned interfaces and +detail the exact changes. + +## A note upfront + +We tried to test this upgrade guide as best as we could by using it to adjust +our official drivers. If you still find breaking changes that causes your +software to crash, please let us and others reading this guide know by sending +over a Pull-Request or notifying us on Gitter. + +Thank you very much :) + + +## Syntactical changes + +#### `tx`, `txid` and `tx_id` becomes `transaction_id` + +To establish better consistency between external interfaces, all usages of +`txid`, `tx` and `tx_id` in data models and HTTP API were renamed to +`transaction_id` or `transaction`. + + +## Breaking Changes to the Data Model + +#### Output amount is now a string + +Planetmint transactions may have multiple inputs and outputs, and each output +has an amount, which is the integral number of the asset being transferred. In +prior versions of Planetmint, the amount was encoded as a number, which on the +face of it is the obvious way to encode an integer. However, as usual the devil +is in the details; JSON, the encoding of choice for Planetmint transactions, +encodes all numbers including integers as floating point. This isn't a problem +for the majority of circumstances where numbers are small, however in some +environments and for some use cases\*, the number may lose precision. + +In order to safeguard against this, amounts are now encoded as strings, and it +is recommended to use a decimal math library (such as +[big.js](https://github.com/MikeMcl/big.js)) when dealing with large numbers in +Javascript. Additionally, numbers are capped at 9e18 to stay comfortably within +the boundary of a 64 bit signed integer. + +\* Try this in the Chrome developer console: `2**60 == 2**60+1`. + + +#### Input `fulfills.txid` is now `transaction_id` + +We renamed a TRANSFER transaction's `inputs.fulfills.txid` to +`inputs.fulfills.transaction_id`. + +#### Input `fulfills.output` is now `output_index` + +We renamed a TRANSFER transaction's `inputs.fulfills.output` to +`inputs.fulfills.output_index`. + +#### Signing payload is now the transaction body + +The signature payload of a Planetmint transaction is now "just" the JSON +serialized body of the transaction. This change is invisible to applications +that do not produce transactions with more than one input. However, prior to +the 1.0 release, transactions with multiple inputs had a special signing +protocol, which included reassembly of the transaction. This was identified as +being unneeded, so now the payload that is signed is always just the serialized +transaction, minus signatures. More details, take a look at the +[Pull-Request](https://github.com/bigchaindb/bigchaindb/pull/1225) introducing +the change or at our updated [Handcrafting +Transactions](https://docs.bigchaindb.com/projects/py-driver/en/latest/handcraft.html) +document. + + +#### Update to Crypto-Conditions version 2 + +Earlier this year the IETF Interledger working group released an [updated +draft](https://tools.ietf.org/html/draft-thomas-crypto-conditions-02) of their +Crypto-Conditions specification. To send transactions to Planetmint v1.0, all +transaction's inputs and outputs need to comply to this new version. + +Several of the language specific implementations have already been updated, +including: + +- [py-crypto-conditions](https://github.com/bigchaindb/cryptoconditions) +- [js-crypto-conditions](https://github.com/interledgerjs/five-bells-condition) +- [java-crypto-conditions](https://github.com/interledger/java-crypto-conditions) + + +If you don't find your preferred language in this list, do not despair but +reach out to us for help on GitHub/Gitter or contact@ipdb.global. + +#### Revamp of Crypto-Conditions signing details + +In order to create a correct fulfillment for an output condition in Planetmint, +we include the conditon URI ("ni:///sha-256;..."), to verify the fulfillment +against. However, the condition URI does not tell you who may sign in order to +create a correct fulfillment. + +For this, we have the `condition.details` object. This is a recursive data structure +which mirrors the n-of-m threshold / ed25519 condition types that we support. + +An example of the new structure is: + +```json +{ + "details": { + "type": "threshold-sha-256", + "threshold": 2, + "subconditions": [ + { + "public_key": "", + "type": "ed25519-sha-256", + }, + { + "public_key": "", + "type": "ed25519-sha-256", + } + ], + }, +} +``` + +#### Transaction version is now 1.0 + +The `version` key in the transaction is now set to `'1.0'`. + + +## Breaking Changes to the HTTP API + +In this section, we'll go over each of the endpoints separately and list the +changes done to them: + + +### `GET /` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#bigchaindb-root-url) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#bigchaindb-root-url) + + +Changes: + +- All notion of `_links` was removed +- `api_v1` is now an object including currently only one further object called + `v1` +- `api.v1` includes links that were originally only available through + `/api/v1/` +- `api.v1` now also includes links to `assets` (a new endpoint) and `outputs` +- `streams_v1`'s link was changed from `streams/valid_tx` to + `streams/valid_transactions` +- `streams_v1` was renamed to `streams` +- Usages of scheme, host and port to API V1's endpoints were removed to allow + for configurations of Planetmint behind reverse proxies + - e.g. `http://example.com:9984/api/v1/transactions` ==> + `/api/v1/transactions` + + +```json +// Old +{ + "_links": { + "api_v1": "http://example.com:9984/api/v1/", + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.10.2/" + }, + "keyring": [ + "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", + "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" + ], + "public_key": "NC8c8rYcAhyKVpx1PCV65CBmyq4YUbLysy3Rqrg8L8mz", + "software": "Planetmint", + "version": "0.10.2" +} + +// New +{ + "api": { + "v1": { + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.11.0.dev/http-client-server-api.html", + "statuses": "/api/v1/statuses/", + "streams": "ws://example.com:9985/api/v1/streams/valid_transactions", + "transactions": "/api/v1/transactions/", + "assets": "/api/v1/assets/", + "outputs": "/api/v1/outputs/" + } + }, + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.11.0.dev/", + "keyring": [ + "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", + "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" + ], + "public_key": "NC8c8rYcAhyKVpx1PCV65CBmyq4YUbLysy3Rqrg8L8mz", + "software": "Planetmint", + "version": "0.11.0.dev" +} +``` + +### `GET /api/v1` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#api-root-endpoint) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#api-root-endpoint) + +Changes: + +- All notion of `_links` was removed +- The response object of `/api/v1` now includes links to `assets` (a new + endpoint) and `outputs` +- `streams_v1`'s link was changed from `streams/valid_tx` to + `streams/valid_transactions` +- `streams_v1` was renamed to `streams` +- Usages of scheme, host and port to API V1's endpoints were removed to allow + for configurations of Planetmint behind reverse proxies + - e.g. `http://example.com:9984/api/v1/transactions` ==> + `/api/v1/transactions` + + +```json +// Old +{ + "_links": { + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html", + "self": "http://example.com:9984/api/v1/", + "statuses": "http://example.com:9984/api/v1/statuses/", + "streams_v1": "ws://example.com:9985/api/v1/streams/valid_tx", + "transactions": "http://example.com:9984/api/v1/transactions/" + } +} + +// New +{ + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.11.0.dev/http-client-server-api.html", + "statuses": "/api/v1/statuses/", + "streams": "ws://example.com:9985/api/v1/streams/valid_transactions", + "transactions": "/api/v1/transactions/", + "assets": "/api/v1/assets/", + "outputs": "/api/v1/outputs/" +} +``` + + +### `GET /api/v1/transactions/{tx_id}` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#get--api-v1-transactions-tx_id) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#get--api-v1-transactions-transaction_id) + + +Changes: + +- Previously this endpoint returned transactions from Planetmint's `BACKLOG` +and from blocks marked as `UNDECIDED`. With version 1.0, this endpoint will +only return transactions included in blocks marked as `VALID`. + + +### `POST /api/v1/transactions` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#post--api-v1-transactions) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#post--api-v1-transactions) + + +Changes: + +- A `Location` HTTP header was included in the endpoint's response to allow + users to check the transaction's status more easily via the + `/statuses?transaction_id` endpoint + + +### `GET /api/v1/outputs` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#get--api-v1-outputs?public_key=public_key) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#get--api-v1-outputs?public_key=public_key) + + +Changes: + +- Reversed the behavior of the `unspent` query parameter to `spent`, implying + the following behavior: + - If `?spent=true`, the response is an array of all spent outputs + associated with a given public key + - If `?spent=false`, response is an array of all NOT YET spent (or + "unspent" outputs associated with a given public key + - If no ``spent=` filter is present in the request, the response is an + array of all outputs associated with a given public key (spent and + unspent) + +Previously the response included a list of relative URLs pointed to +transations' outputs: + +```json +// Old +[ + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/0" +] + +// New +[ + { + "output_index": 0, + "transaction_id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e" + } +] +``` + +In the future, we're planning to [upgrade this endpoint +further](https://github.com/bigchaindb/bigchaindb/blob/99499b1f8783719a082813912ac9a0d363ae278f/bdb-ip.md#6-a-new-outputs-endpoint) +to meet the requirements of [our +users](https://github.com/bigchaindb/bigchaindb/issues/1227#issuecomment-307297473). + + +### `GET /api/v1/statuses?tx_id` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#get--api-v1-statuses?tx_id=tx_id) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#get--api-v1-statuses?transaction_id=transaction_id) + + +Changes: + +- All notion of `_links` was removed. In case of querying the status of a + transaction already included in a block marked `VALID`, no `_links` object is + provided anymore. The response object now only contains a single key value + pair named `status` +- The query parameter `tx_id` was renamed to `transaction_id`, e.g. `GET + /api/v1/statuses?transaction_id=` + + +```json +// Old +{ + "status": "valid", + "_links": { + "tx": "/transactions/04c00267af82c161b4bf2ad4a47d1ddbfeb47eef1a14b8d51f37d6ee00ea5cdd" + } +} + +// New +{ + "status": "valid", +} +``` + + +### `GET /api/v1/statuses?block_id` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#get--api-v1-statuses?block_id=block_id) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#get--api-v1-statuses?block_id=block_id) + + +Changes: + +- All notion of `_links` was removed. The response object now only contains a + single key value pair named `status` + + +```json +// Old +{ + "status": "valid", + "_links": { + "tx": "/transactions/04c00267af82c161b4bf2ad4a47d1ddbfeb47eef1a14b8d51f37d6ee00ea5cdd" + } +} + +// New +{ + "status": "valid", +} +``` + + +### `GET /api/v1/blocks?tx_id` + +Documentation: + +- [Old](https://docs.bigchaindb.com/projects/server/en/v0.10.2/http-client-server-api.html#get--api-v1-blocks?tx_id=tx_id&status=UNDECIDED|VALID|INVALID) +- [New](https://docs.bigchaindb.com/projects/server/en/v1.0.0/http-client-server-api.html#get--api-v1-blocks?transaction_id=transaction_id&status=UNDECIDED|VALID|INVALID) + + +Changes: + +- The query parameter `tx_id` was renamed to `transaction_id`, e.g. `GET + /api/v1/blocks?transaction_id` + + +## Breaking Changes to the WebSocket Event Stream API + +In the event object sent to a listener, `tx_id` was renamed to +`transaction_id`. + +```json +// Old +{ + "tx_id": "", + "asset_id": "", + "block_id": "" +} + +// New +{ + "transaction_id": "", + "asset_id": "", + "block_id": "" +} +``` diff --git a/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml b/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml new file mode 100644 index 0000000..cb8db41 --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-ext-conn-svc.yaml @@ -0,0 +1,27 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + # Name of Planetmint instance you are trying to connect to + # name: "bdb-instance-0" + name: "" + namespace: default +spec: + ports: +spec: + ports: + - port: 26656 + name: p2p + - port: 26657 + name: pubkey + - port: 9986 + name: nginx + type: ExternalName + # FQDN of remote cluster/NGINX instance + #externalName: "nginx-instance-for-bdb-instance-0.westeurope.cloudapp.azure.com" + externalName: "" + diff --git a/k8s/bigchaindb/bigchaindb-pv.yaml b/k8s/bigchaindb/bigchaindb-pv.yaml new file mode 100644 index 0000000..8a6aa18 --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-pv.yaml @@ -0,0 +1,46 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +######################################################### +# This YAML section desribes a k8s PV for tendermint db # +######################################################### +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-tm-db +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 20Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: tendermint-db +--- +############################################################## +# This YAML section desribes a k8s PV for Tendermint config # +############################################################## +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-tm-configdb +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: tendermint-config-db diff --git a/k8s/bigchaindb/bigchaindb-pvc.yaml b/k8s/bigchaindb/bigchaindb-pvc.yaml new file mode 100644 index 0000000..5e22eb8 --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-pvc.yaml @@ -0,0 +1,36 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +########################################################## +# This section file desribes a k8s pvc for tendermint db # +########################################################## +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: tendermint-db-claim + annotations: + volume.beta.kubernetes.io/storage-class: tendermint-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +################################################################ +# This YAML section desribes a k8s pvc for tendermint configdb # +################################################################ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: tendermint-config-db-claim + annotations: + volume.beta.kubernetes.io/storage-class: tendermint-config-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/k8s/bigchaindb/bigchaindb-sc.yaml b/k8s/bigchaindb/bigchaindb-sc.yaml new file mode 100644 index 0000000..49dbbfe --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-sc.yaml @@ -0,0 +1,36 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +################################################################### +# This YAML section desribes a StorageClass for the tendermint db # +################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: tendermint-db +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] + location: + # If you have created a different storage account e.g. for Premium Storage + storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed +--- +######################################################################### +# This YAML section desribes a StorageClass for the tendermint configdb # +######################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: tendermint-config-db +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] + location: + # If you have created a different storage account e.g. for Premium Storage + storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed diff --git a/k8s/bigchaindb/bigchaindb-ss.yaml b/k8s/bigchaindb/bigchaindb-ss.yaml new file mode 100644 index 0000000..3bcf74b --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-ss.yaml @@ -0,0 +1,295 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +################################################################################# +# This YAML file desribes a StatefulSet with a service for running and exposing # +# a Tendermint instance. It depends on the tendermint-config-db-claim # +# and tendermint-db-claim k8s pvc. # +################################################################################# + +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: bdb-instance-0-ss + namespace: default +spec: + serviceName: bdb-instance-0 + replicas: 1 + template: + metadata: + name: bdb-instance-0-ss + labels: + app: bdb-instance-0-ss + spec: + restartPolicy: Always + volumes: + - name: bdb-data + persistentVolumeClaim: + claimName: tendermint-db-claim + - name: bdb-config-data + persistentVolumeClaim: + claimName: tendermint-config-db-claim + - name: bdb-certs + secret: + secretName: bdb-certs + defaultMode: 0400 + - name: ca-auth + secret: + secretName: ca-auth + defaultMode: 0400 + containers: + # Treating bigchaindb+ nginx + tendermint as a POD because they should not + # exist without each other + # Nginx container for hosting public key of this ndoe + - name: nginx + imagePullPolicy: Always + image: bigchaindb/nginx_pub_key_access:2.2.2 + env: + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-pub-key-access + ports: + - containerPort: 9986 + name: bdb-pk-access + volumeMounts: + - name: bdb-config-data + mountPath: /usr/share/nginx + readOnly: true + #Tendermint container + - name: tendermint + imagePullPolicy: Always + image: bigchaindb/tendermint:2.2.2 + env: + - name: TM_PERSISTENT_PEERS + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-persistent-peers + - name: TM_VALIDATOR_POWER + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-validator-power + - name: TM_VALIDATORS + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-validators + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-pub-key-access + - name: TM_GENESIS_TIME + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-genesis-time + - name: TM_CHAIN_ID + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-chain-id + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-p2p-port + - name: TM_INSTANCE_NAME + valueFrom: + configMapKeyRef: + name: vars + key: bdb-instance-name + - name: TMHOME + value: /tendermint + - name: TM_PROXY_APP + valueFrom: + configMapKeyRef: + name: vars + key: bdb-instance-name + - name: TM_ABCI_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-abci-port + - name: TM_RPC_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-rpc-port + resources: + limits: + cpu: 1 + memory: 5G + volumeMounts: + - name: bdb-data + mountPath: /tendermint + - name: bdb-config-data + mountPath: /tendermint_node_data + ports: + - containerPort: 26656 + name: p2p + - containerPort: 26657 + name: rpc + livenessProbe: + exec: + command: + - /bin/bash + - "-c" + - | + curl -s --fail --max-time 10 "http://${TM_INSTANCE_NAME}:${TM_RPC_PORT}/abci_info" > /dev/null && \ + curl -s --fail --max-time 10 "http://${TM_INSTANCE_NAME}:${TM_RPC_PORT}/status" > /dev/null + ERR=$? + if [ "$ERR" == 28 ]; then + exit 1 + elif [[ $(curl --max-time 10 "http://${TM_INSTANCE_NAME}:${TM_RPC_PORT}/abci_info" | jq -r ".error.code") == -32603 ]]; then + exit 1 + elif [ "$ERR" != 0 ]; then + exit 1 + else + exit 0 + fi + initialDelaySeconds: 60 + periodSeconds: 60 + failureThreshold: 3 + timeoutSeconds: 15 + # Planetmint container + - name: bigchaindb + image: bigchaindb/bigchaindb:2.2.2 + imagePullPolicy: Always + args: + - start + env: + - name: PLANETMINT_DATABASE_HOST + valueFrom: + configMapKeyRef: + name: vars + key: mdb-instance-name + - name: PLANETMINT_DATABASE_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: PLANETMINT_DATABASE_BACKEND + value: "localmongodb" + - name: PLANETMINT_DATABASE_NAME + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-database-name + - name: PLANETMINT_SERVER_BIND + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-server-bind + - name: PLANETMINT_WSSERVER_HOST + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-interface + - name: PLANETMINT_WSSERVER_ADVERTISED_HOST + valueFrom: + configMapKeyRef: + name: vars + key: node-fqdn + - name: PLANETMINT_WSSERVER_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-port + - name: PLANETMINT_WSSERVER_ADVERTISED_PORT + valueFrom: + configMapKeyRef: + name: vars + key: node-frontend-port + - name: PLANETMINT_WSSERVER_ADVERTISED_SCHEME + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-wsserver-advertised-scheme + - name: PLANETMINT_DATABASE_MAXTRIES + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-database-maxtries + - name: PLANETMINT_DATABASE_CONNECTION_TIMEOUT + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-database-connection-timeout + - name: PLANETMINT_LOG_LEVEL_CONSOLE + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-log-level + - name: PLANETMINT_DATABASE_SSL + value: "true" + - name: PLANETMINT_DATABASE_CA_CERT + value: /etc/bigchaindb/ca/ca.pem + - name: PLANETMINT_DATABASE_CRLFILE + value: /etc/bigchaindb/ca/crl.pem + - name: PLANETMINT_DATABASE_CERTFILE + value: /etc/bigchaindb/ssl/bdb-instance.pem + - name: PLANETMINT_DATABASE_KEYFILE + value: /etc/bigchaindb/ssl/bdb-instance.key + - name: PLANETMINT_DATABASE_LOGIN + valueFrom: + configMapKeyRef: + name: bdb-config + key: bdb-user + - name: PLANETMINT_TENDERMINT_HOST + valueFrom: + configMapKeyRef: + name: vars + key: bdb-instance-name + - name: PLANETMINT_TENDERMINT_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-rpc-port + command: + - bash + - "-c" + - | + curl -s --fail "http://${PLANETMINT_TENDERMINT_HOST}:9986/pub_key.json" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + sleep 30 + curl -s --fail "http://${PLANETMINT_TENDERMINT_HOST}:9986/pub_key.json" > /dev/null + ERR=$? + echo "Waiting for Tendermint instance." + done + bigchaindb -l DEBUG start + ports: + - containerPort: 9984 + protocol: TCP + name: bdb-port + - containerPort: 9985 + protocol: TCP + name: bdb-ws-port + - containerPort: 26658 + protocol: TCP + name: bdb-abci-port + volumeMounts: + - name: bdb-certs + mountPath: /etc/bigchaindb/ssl/ + readOnly: true + - name: ca-auth + mountPath: /etc/bigchaindb/ca/ + readOnly: true + resources: + limits: + cpu: 200m + memory: 2G + livenessProbe: + httpGet: + path: / + port: bdb-port + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 15 diff --git a/k8s/bigchaindb/bigchaindb-svc.yaml b/k8s/bigchaindb/bigchaindb-svc.yaml new file mode 100644 index 0000000..353b63c --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-svc.yaml @@ -0,0 +1,42 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: bdb-instance-0 + namespace: default + labels: + name: bdb-instance-0 +spec: + selector: + app: bdb-instance-0-ss + ports: + - port: 9984 + targetPort: 9984 + name: bdb-api-port + protocol: TCP + - port: 9985 + targetPort: 9985 + name: bdb-ws-port + protocol: TCP + - port: 26658 + targetPort: 26658 + name: tm-abci-port + protocol: TCP + - port: 26656 + targetPort: 26656 + name: tm-p2p-port + protocol: TCP + - port: 26657 + targetPort: 26657 + name: tm-rpc-port + protocol: TCP + - port: 9986 + targetPort: 9986 + name: pub-key-access + protocol: TCP + type: ClusterIP + clusterIP: None diff --git a/k8s/bigchaindb/nginx_container/Dockerfile b/k8s/bigchaindb/nginx_container/Dockerfile new file mode 100644 index 0000000..ec5d2b6 --- /dev/null +++ b/k8s/bigchaindb/nginx_container/Dockerfile @@ -0,0 +1,12 @@ +FROM nginx:stable +LABEL maintainer "contact@ipdb.global" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /etc/nginx/conf.d/access_pub_key.conf +COPY nginx_entrypoint.bash / +VOLUME /usr/share/nginx +EXPOSE 9986 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/bigchaindb/nginx_container/README.md b/k8s/bigchaindb/nginx_container/README.md new file mode 100644 index 0000000..0983c56 --- /dev/null +++ b/k8s/bigchaindb/nginx_container/README.md @@ -0,0 +1,26 @@ + + +## Nginx container for hosting public key for a tendermint instance + + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + +### Step 2: Run the container + +``` +docker run \ + --name=tendermint_instance_pub_key \ + --env TM_PUB_KEY_ACCESS_PORT='' + --publish=: \ + --volume=:/usr/share/nginx \ + bigchaindb/nginx_pub_key_access: +``` diff --git a/k8s/bigchaindb/nginx_container/docker_build_and_push.bash b/k8s/bigchaindb/nginx_container/docker_build_and_push.bash new file mode 100755 index 0000000..522f99e --- /dev/null +++ b/k8s/bigchaindb/nginx_container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/nginx_pub_key_access:2.2.2 . + +docker push bigchaindb/nginx_pub_key_access:2.2.2 diff --git a/k8s/bigchaindb/nginx_container/nginx.conf.template b/k8s/bigchaindb/nginx_container/nginx.conf.template new file mode 100644 index 0000000..90251dd --- /dev/null +++ b/k8s/bigchaindb/nginx_container/nginx.conf.template @@ -0,0 +1,10 @@ +# Serve the public key for a tendermint instance + +server { + listen PUBLIC_KEY_ACCESS_PORT default_server; + listen [::]:PUBLIC_KEY_ACCESS_PORT default_server ipv6only=on; + location / { + root /usr/share/nginx/; + autoindex on; + } +} diff --git a/k8s/bigchaindb/nginx_container/nginx_entrypoint.bash b/k8s/bigchaindb/nginx_container/nginx_entrypoint.bash new file mode 100755 index 0000000..611aad1 --- /dev/null +++ b/k8s/bigchaindb/nginx_container/nginx_entrypoint.bash @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +# Tendermint public key access port +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` + +if [[ -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting}" ]]; then + exit 1 +else + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" +fi + +NGINX_CONF_FILE=/etc/nginx/conf.d/access_pub_key.conf + +# configure the access_pub_key file with env variable(s) +sed -i "s|PUBLIC_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} + +cat /etc/nginx/conf.d/access_pub_key.conf +# start nginx +echo "INFO: starting nginx..." +exec nginx -g "daemon off;" diff --git a/k8s/bigchaindb/tendermint_container/Dockerfile b/k8s/bigchaindb/tendermint_container/Dockerfile new file mode 100644 index 0000000..1404f46 --- /dev/null +++ b/k8s/bigchaindb/tendermint_container/Dockerfile @@ -0,0 +1,10 @@ +FROM tendermint/tendermint:v0.31.5 +LABEL maintainer "contact@ipdb.global" +WORKDIR / +USER root +RUN apk --update add bash +COPY genesis.json.template /etc/tendermint/genesis.json +COPY tendermint_entrypoint.bash / +VOLUME /tendermint /tendermint_node_data +EXPOSE 26656 26657 +ENTRYPOINT ["/tendermint_entrypoint.bash"] diff --git a/k8s/bigchaindb/tendermint_container/README.md b/k8s/bigchaindb/tendermint_container/README.md new file mode 100644 index 0000000..10799e6 --- /dev/null +++ b/k8s/bigchaindb/tendermint_container/README.md @@ -0,0 +1,36 @@ + + +## Tendermint container used for BFT replication and consensus + + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + +### Step 2: Run the container + +``` +docker run \ + --name=tendermint \ + --env TM_PUB_KEY_ACCESS_PORT= \ + --env TM_PERSISTENT_PEERS= \ + --env TM_VALIDATOR_POWER= \ + --env TM_VALIDATORS= \ + --env TM_GENESIS_TIME= \ + --env TM_CHAIN_ID= \ + --env TM_P2P_PORT= \ + --env TMHOME= \ + --env TM_PROXY_APP= \ + --publish=: \ + --publish=: \ + --volume :/tendermint \ + --volume=:/tendermint_node_data \ + bigchaindb/tendermint: +``` diff --git a/k8s/bigchaindb/tendermint_container/docker_build_and_push.bash b/k8s/bigchaindb/tendermint_container/docker_build_and_push.bash new file mode 100755 index 0000000..4dcf883 --- /dev/null +++ b/k8s/bigchaindb/tendermint_container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/tendermint:2.2.2 . + +docker push bigchaindb/tendermint:2.2.2 diff --git a/k8s/bigchaindb/tendermint_container/genesis.json.template b/k8s/bigchaindb/tendermint_container/genesis.json.template new file mode 100644 index 0000000..7b726b6 --- /dev/null +++ b/k8s/bigchaindb/tendermint_container/genesis.json.template @@ -0,0 +1,6 @@ +{ + "genesis_time": TM_GENESIS_TIME, + "chain_id": TM_CHAIN_ID, + "validators": [], + "app_hash": "" +} diff --git a/k8s/bigchaindb/tendermint_container/tendermint_entrypoint.bash b/k8s/bigchaindb/tendermint_container/tendermint_entrypoint.bash new file mode 100755 index 0000000..b3c06e0 --- /dev/null +++ b/k8s/bigchaindb/tendermint_container/tendermint_entrypoint.bash @@ -0,0 +1,141 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +# Cluster vars +tm_persistent_peers=`printenv TM_PERSISTENT_PEERS` +tm_validators=`printenv TM_VALIDATORS` +tm_validator_power=`printenv TM_VALIDATOR_POWER` +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` +tm_genesis_time=`printenv TM_GENESIS_TIME` +tm_chain_id=`printenv TM_CHAIN_ID` +tm_p2p_port=`printenv TM_P2P_PORT` + + +# tendermint node vars +tmhome=`printenv TMHOME` +tm_proxy_app=`printenv TM_PROXY_APP` +tm_abci_port=`printenv TM_ABCI_PORT` +tm_instance_name=`printenv TM_INSTANCE_NAME` + +# Container vars +RETRIES=0 +CANNOT_INITIATLIZE_INSTANCE='Cannot start instance, if initial validator(s) are unreachable.' + + +# sanity check +if [[ -z "${tm_persistent_peers:?TM_PERSISTENT_PEERS not specified. Exiting!}" || \ + -z "${tm_validators:?TM_VALIDATORS not specified. Exiting!}" || \ + -z "${tm_validator_power:?TM_VALIDATOR_POWER not specified. Exiting!}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_genesis_time:?TM_GENESIS_TIME not specified. Exiting!}" || \ + -z "${tm_chain_id:?TM_CHAIN_ID not specified. Exiting!}" || \ + -z "${tmhome:?TMHOME not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" || \ + -z "${tm_abci_port:?TM_ABCI_PORT not specified. Exiting! }" || \ + -z "${tm_instance_name:?TM_INSTANCE_NAME not specified. Exiting! }" ]]; then + echo "Missing required enviroment variables." + exit 1 +else + echo tm_persistent_peers="$TM_PERSISTENT_PEERS" + echo tm_validators="$TM_VALIDATORS" + echo tm_validator_power="$TM_VALIDATOR_POWER" + echo tm_pub_key_access_port="$TM_PUB_KEY_ACCESS_PORT" + echo tm_genesis_time="$TM_GENESIS_TIME" + echo tm_chain_id="$TM_CHAIN_ID" + echo tmhome="$TMHOME" + echo tm_p2p_port="$TM_P2P_PORT" + echo tm_abci_port="$TM_ABCI_PORT" + echo tm_instance_name="$TM_INSTANCE_NAME" +fi + +# copy template +mkdir -p /tendermint/config +cp /etc/tendermint/genesis.json /tendermint/config/genesis.json + +TM_GENESIS_FILE=/tendermint/config/genesis.json +TM_PUB_KEY_DIR=/tendermint_node_data + +# configure the nginx.conf file with env variables +sed -i "s|TM_GENESIS_TIME|\"${tm_genesis_time}\"|g" ${TM_GENESIS_FILE} +sed -i "s|TM_CHAIN_ID|\"${tm_chain_id}\"|g" ${TM_GENESIS_FILE} + +if [ ! -f /tendermint/config/priv_validator.json ]; then + tendermint gen_validator > /tendermint/config/priv_validator.json + # pub_key.json will be served by the nginx container + cat /tendermint/config/priv_validator.json + cat /tendermint/config/priv_validator.json \ + | jq ".Key.pub_key" > "$TM_PUB_KEY_DIR"/pub_key.json +fi + +if [ ! -f /tendermint/config/node_key.json ]; then + tendermint gen_node_key > "$TM_PUB_KEY_DIR"/address +fi + +# fill genesis file with validators +IFS=',' read -ra VALS_ARR <<< "$TM_VALIDATORS" +IFS=',' read -ra VAL_POWERS_ARR <<< "$TM_VALIDATOR_POWER" +if [ ${#VALS_ARR[@]} -ne ${#VAL_POWERS_ARR[@]} ]; then + echo "Invalid configuration of Validator(s) and Validator Power(s)" + exit 1 +fi +for i in "${!VALS_ARR[@]}"; do + # wait until validator generates priv/pub key pair + set +e + echo Validator: "${VALS_ARR[$i]}" + echo Validator Power: "${VAL_POWERS_ARR[$i]}" + echo "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + RETRIES=$((RETRIES+1)) + if [ $RETRIES -eq 10 ]; then + echo "${CANNOT_INITIATLIZE_INSTANCE}" + exit 1 + fi + # 300(30 * 10(retries)) second timeout before container dies if it cannot find initial peers + sleep 30 + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + echo "Cannot get public key for Tendermint instance: ${VALS_ARR[$i]}" + done + set -e + # add validator to genesis file along with its pub_key + curl -s "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" | jq ". as \$k | {pub_key: \$k, power: \"${VAL_POWERS_ARR[$i]}\", name: \"${VALS_ARR[$i]}\"}" > pub_validator.json + cat /tendermint/config/genesis.json \ + | jq ".validators |= .+ [$(cat pub_validator.json)]" > tmpgenesis \ + && mv tmpgenesis /tendermint/config/genesis.json + rm pub_validator.json +done + +# construct persistent peers +IFS=',' read -ra PEERS_ARR <<< "$tm_persistent_peers" +peers=() +for s in "${PEERS_ARR[@]}"; do + echo "http://$s:$tm_pub_key_access_port/address" + curl -s --fail "http://$s:$tm_pub_key_access_port/address" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + RETRIES=$((RETRIES+1)) + if [ $RETRIES -eq 10 ]; then + echo "${CANNOT_INITIATLIZE_INSTANCE}" + exit 1 + fi + # 300(30 * 10(retries)) second timeout before container dies if it cannot find initial peers + sleep 30 + curl -s --fail "http://$s:$tm_pub_key_access_port/address" > /dev/null + ERR=$? + echo "Cannot get address for Tendermint instance: ${s}" + done + peer_addr=$(curl -s "http://$s:$tm_pub_key_access_port/address") + peers+=("$peer_addr@$s:$tm_p2p_port") +done +peers=$(IFS=','; echo "${peers[*]}") + +# start nginx +echo "INFO: starting tendermint..." +exec tendermint node --p2p.persistent_peers="$peers" --moniker="$tm_instance_name" --proxy_app="tcp://$tm_proxy_app:$tm_abci_port" --consensus.create_empty_blocks=false --p2p.pex=false diff --git a/k8s/configuration/config-map.yaml b/k8s/configuration/config-map.yaml new file mode 100644 index 0000000..2f1058c --- /dev/null +++ b/k8s/configuration/config-map.yaml @@ -0,0 +1,179 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +## Note: data values do NOT have to be base64-encoded in this file. + +## vars is common environment variables for this BigchaindB node +apiVersion: v1 +kind: ConfigMap +metadata: + name: vars + namespace: default +data: + # node-fqdn is the DNS name registered for your HTTPS certificate. + node-fqdn: "bdb.example.com" + + # node-frontend-port is the port number on which this node's services + # are available to external clients. + node-frontend-port: "443" + + # node-health-check-port is the port number on which an external load + # balancer can check the status/liveness of the external/public server. + # In our deployment, Kubernetes sends 'livenessProbes' to this port and + # interprets a successful response as a 'healthy' service. + node-health-check-port: "8888" + + # node-dns-server-ip is the IP of the DNS server. A Kubernetes deployment + # always has a DNS server (kube-dns). + node-dns-server-ip: "" + + # mdb-instance-name is the name of the MongoDB instance in this Kubernetes cluster. + mdb-instance-name: "" + + # ngx-instance-name is the name of the NGINX instance in this Kubernetes cluster. + ngx-instance-name: "" + + # openresty-instance-name is the name of the OpenResty instance in this + # Kubernetes cluster. + openresty-instance-name: "" + + # bdb-instance-name is the name of the Planetmint instance in this Kubernetes cluster. + bdb-instance-name: "" + + # mdb-mon-instance-name is the name of the MongoDB Monitoring Agent instance + # in this Kubernetes cluster. + mdb-mon-instance-name: "" + + # ngx-mdb-instance-name is the FQDN of the MongoDB instance in this + # Kubernetes cluster. + ngx-mdb-instance-name: ".default.svc.cluster.local" + + # ngx-openresty-instance-name is the FQDN of the OpenResty instance in this + # Kubernetes cluster. + ngx-openresty-instance-name: ".default.svc.cluster.local" + + # ngx-bdb-instance-name is the FQDN of the Planetmint instance in this + # Kubernetes cluster. + ngx-bdb-instance-name: ".default.svc.cluster.local" + + # mongodb-backend-port is the port on which MongoDB is actually + # available/listening for requests. + mongodb-backend-port: "27017" + + # TODO: Change `openresty` to `apicast` + # openresty-backend-port is the port number on which OpenResty is listening + # for requests. This is used by the NGINX instance to forward the requests to + # the right port, and by OpenResty instance to bind to the correct port to + # receive requests from NGINX instance. + openresty-backend-port: "8080" + + # Planetmint configuration parameters + # Refer https://docs.bigchaindb.com/projects/server/en/latest/server-reference/configuration.html + + # bigchaindb-api-port is the port number on which Planetmint is listening + # for HTTP requests. + bigchaindb-api-port: "9984" + + # bigchaindb-server-bind is the socket where Planetmint binds for API + # requests. + bigchaindb-server-bind: "0.0.0.0:9984" + + # bigchaindb-ws-port and bigchaindb-ws-interface form the socket where + # Planetmint binds for Websocket connections. + bigchaindb-ws-port: "9985" + bigchaindb-ws-interface: "0.0.0.0" + + # bigchaindb-database-name is the database collection used by Planetmint with + # the MongoDB backend. + bigchaindb-database-name: "bigchain" + + # bigchaindb-wsserver-advertised-scheme is the protocol used to access the + # WebSocket API in Planetmint; can be 'ws' or 'wss' (default). + bigchaindb-wsserver-advertised-scheme: "wss" + + # Optional: Optimize storage engine(wired tiger) + # cache size. e.g. (2048MB, 2GB, 1TB), otherwise + # it will use the default cache size; i.e. max((50% RAM - 1GB), 256MB) + storage-engine-cache-size: "" + + # POST API authorization mode [threescale | secrete-token] + authorization-mode: "threescale" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: bdb-config + namespace: default +data: + # Planetmint instance authentication user name + bdb-user: "" + + # bigchaindb-database-maxtries is the maximum number of times that Planetmint + # will try to establish a connection with the database backend. + # If it is set to 0, then it will try forever. + bigchaindb-database-maxtries: "3" + + # bigchaindb-database-connection-timeout is the maximum number of + # milliseconds that Planetmint will wait before closing the connection while + # connecting to the database backend. + bigchaindb-database-connection-timeout: "5000" + + # bigchaindb-log-level is the log level used to log to the console. + bigchaindb-log-level: "debug" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tendermint-config + namespace: default +data: + # bdb-persistent-peers is the list of all the peers in the network. + bdb-persistent-peers: "<',' separated list of all tendermint peers in the network>" + + # bdb-validators is the list of all validators in the network. + bdb-validators: "<',' separated list of all validators in the network>" + + # bdb-validator-power is the validators voting power, make sure the order and + # the number of nodes in tm-validator-power and tm-validators is the same. + bdb-validator-power: "<',' separated list of validator power of each node in the network>" + + # bdb-genesis-time is the official time of blockchain start. + # example: 0001-01-01T00:00:00Z + bdb-genesis-time: "" + + # bdb-chain-id is the ID of the blockchain. Must be unique for every blockchain. + # example: test-chain-KPI1Ud + bdb-chain-id: "" + + # bdb-abci-port is used by Tendermint Core for ABCI traffic. Planetmint nodes + # use that internally. + bdb-abci-port: "26658" + + # bdb-p2p-port is used by Tendermint Core to communicate with + # other peers in the network. This port is accessible publicly. + bdb-p2p-port: "26656" + + # bdb-rpc-port is used by Tendermint Core to rpc. Planetmint nodes + # use this port internally. + bbd-rpc-port: "26657" + + # bdb-pub-key-access is the port number used to host/publish the + # public key of the tendemrint node in this cluster. + bdb-pub-key-access: "9986" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mdb-config + namespace: default +data: + # User name for MongoDB adminuser + mdb-admin-username: "" + + # MongoDB monitoring agent authentication user name + mdb-mon-user: "" diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml new file mode 100644 index 0000000..60669f7 --- /dev/null +++ b/k8s/configuration/secret.yaml @@ -0,0 +1,118 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# All secret data should be base64 encoded before embedding them here. +# Short strings can be encoded using, e.g. +# echo "secret string" | base64 -w 0 > secret.string.b64 +# Files (e.g. certificates) can be encoded using, e.g. +# cat cert.pem | base64 -w 0 > cert.pem.b64 +# then copy the contents of cert.pem.b64 (for example) below. +# Ref: https://kubernetes.io/docs/concepts/configuration/secret/ +# Unused values can be set to "" + +apiVersion: v1 +kind: Secret +metadata: + name: cloud-manager-credentials + namespace: default +type: Opaque +data: + # Base64-encoded Project ID + # Project ID used by MongoDB deployment + group-id: "" + # Base64-encoded MongoDB Agent API Key for the group + agent-api-key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-instance.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-mon-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-mon-instance.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: bdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded Planetmint instance certificate + bdb-instance.pem: "" + # Base64-encoded private key (.key) + bdb-instance.key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: nginx-secret-header + namespace: default +type: Opaque +data: + # Base64-encoded secret token to authorize POST requests + secret-token: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-certs + namespace: default +type: Opaque +data: + # Base64-encoded HTTPS private key + cert.key: "" + # Base64-encoded HTTPS certificate chain + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert if from DigiCert, download "Best format for nginx". + cert.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: threescale-credentials + namespace: default +type: Opaque +data: + secret-token: "" + service-id: "" + version-header: "" + service-token: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ca-auth + namespace: default +type: Opaque +data: + # CA used to issue members/client certificates + # Base64-encoded CA certificate (ca.crt) + ca.pem: "" + crl.pem: "" +--- +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-config + namespace: default +type: Opaque +data: + # Password for for MongoDB adminuser + mdb-admin-password: "" diff --git a/k8s/dev-setup/bigchaindb.yaml b/k8s/dev-setup/bigchaindb.yaml new file mode 100644 index 0000000..ede6e33 --- /dev/null +++ b/k8s/dev-setup/bigchaindb.yaml @@ -0,0 +1,98 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: bdb + namespace: default + labels: + name: bdb +spec: + selector: + app: bdb-dep + ports: + - port: 9984 + targetPort: 9984 + name: bdb-api-port + protocol: TCP + - port: 9985 + targetPort: 9985 + name: bdb-ws-port + protocol: TCP + type: ClusterIP + clusterIP: None +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: bdb-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: bdb-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: bigchaindb + image: bigchaindb/bigchaindb:2.2.2 + imagePullPolicy: Always + args: + - start + env: + - name: PLANETMINT_DATABASE_HOST + value: mdb + - name: PLANETMINT_DATABASE_PORT + value: "27017" + - name: PLANETMINT_DATABASE_REPLICASET + value: bigchain-rs + - name: PLANETMINT_DATABASE_BACKEND + value: mongodb + - name: PLANETMINT_DATABASE_NAME + value: bigchain + - name: PLANETMINT_SERVER_BIND + value: "0.0.0.0:9984" + - name: PLANETMINT_WSSERVER_HOST + value: "0.0.0.0" + - name: PLANETMINT_WSSERVER_PORT + value: "9985" + - name: PLANETMINT_KEYPAIR_PUBLIC + value: "EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ" + - name: PLANETMINT_KEYPAIR_PRIVATE + value: "3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm" + - name: PLANETMINT_DATABASE_MAXTRIES + value: "3" + - name: PLANETMINT_DATABASE_CONNECTION_TIMEOUT + value: "120" + - name: PLANETMINT_LOG_LEVEL_CONSOLE + value: debug + ports: + - containerPort: 9984 + hostPort: 9984 + name: bdb-port + protocol: TCP + - containerPort: 9985 + hostPort: 9985 + name: bdb-ws-port + protocol: TCP + resources: + limits: + cpu: 200m + memory: 768Mi + livenessProbe: + httpGet: + path: / + port: 9984 + initialDelaySeconds: 15 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: / + port: 9984 + initialDelaySeconds: 15 + timeoutSeconds: 10 + restartPolicy: Always diff --git a/k8s/dev-setup/mongo.yaml b/k8s/dev-setup/mongo.yaml new file mode 100644 index 0000000..539592d --- /dev/null +++ b/k8s/dev-setup/mongo.yaml @@ -0,0 +1,54 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: mdb + namespace: default + labels: + name: mdb +spec: + selector: + app: mdb-ss + ports: + - port: 27017 + targetPort: 27017 + protocol: TCP + name: mdb-svc-port + type: ClusterIP + clusterIP: None +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: mdb-ss + namespace: default +spec: + serviceName: mdb + replicas: 1 + template: + metadata: + name: mdb-ss + labels: + app: mdb-ss + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongodb + image: mongo:3.4.4 + imagePullPolicy: Always + args: + - --replSet + - bigchain-rs + ports: + - containerPort: 27017 + hostPort: 27017 + protocol: TCP + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always diff --git a/k8s/dev-setup/nginx-http.yaml b/k8s/dev-setup/nginx-http.yaml new file mode 100644 index 0000000..6dae46d --- /dev/null +++ b/k8s/dev-setup/nginx-http.yaml @@ -0,0 +1,88 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: ngx-http + namespace: default + labels: + name: ngx-http + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + # Ref: https://kubernetes.io/docs/tutorials/services/source-ip/ + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-http-dep + ports: + - port: 80 + targetPort: 80 + name: ngx-public-bdb-port-http + protocol: TCP + - port: 27017 + targetPort: 27017 + name: ngx-public-mdb-port + protocol: TCP + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-http-dep + namespace: default +spec: + replicas: 1 + template: + metadata: + name: ngx-http-dep + labels: + app: ngx-http-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx-http + image: bigchaindb/nginx_http:1.0 + imagePullPolicy: Always + env: + - name: CLUSTER_FRONTEND_PORT + value: "80" + - name: HEALTH_CHECK_PORT + value: "8080" + - name: DNS_SERVER + value: "10.0.0.10" + - name: MONGODB_FRONTEND_PORT + value: "27017" + - name: MONGODB_BACKEND_HOST + value: "mdb.default.svc.cluster.local" + - name: MONGODB_BACKEND_PORT + value: "27017" + - name: PLANETMINT_BACKEND_HOST + value: "bdb.default.svc.cluster.local" + - name: PLANETMINT_API_PORT + value: "9984" + - name: PLANETMINT_WS_PORT + value: "9985" + ports: + - containerPort: 27017 + protocol: TCP + - containerPort: 80 + protocol: TCP + - containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always diff --git a/k8s/dev-setup/nginx-https.yaml b/k8s/dev-setup/nginx-https.yaml new file mode 100644 index 0000000..95168fb --- /dev/null +++ b/k8s/dev-setup/nginx-https.yaml @@ -0,0 +1,126 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Secret +metadata: + name: https-certs + namespace: default +type: Opaque +data: + # Base64-encoded HTTPS private key + cert.key: "" + # Base64-encoded HTTPS certificate chain + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert if from DigiCert, download "Best format for nginx". + cert.pem: "" + service-id: "" + version-header: "" + provider-key: "" +--- +apiVersion: v1 +kind: Service +metadata: + name: openresty + namespace: default + labels: + name: openresty + annotations: + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: openresty-dep + ports: + - port: 80 + targetPort: 80 + name: openresty-svc-port + protocol: TCP + type: ClusterIP + clusterIP: None +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: openresty-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: openresty-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx-openresty + image: bigchaindb/nginx_3scale:2.0 + imagePullPolicy: Always + env: + - name: DNS_SERVER + value: "10.0.0.10" + - name: OPENRESTY_FRONTEND_PORT + value: "80" + - name: PLANETMINT_BACKEND_HOST + value: "bdb.default.svc.cluster.local" + - name: PLANETMINT_API_PORT + value: "9984" + ports: + - containerPort: 80 + protocol: TCP + name: openresty-port + volumeMounts: + - name: threescale-credentials + mountPath: /usr/local/openresty/nginx/conf/threescale + readOnly: true + livenessProbe: + httpGet: + path: / + port: openresty-port + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always + volumes: + - name: threescale-credentials + secret: + secretName: threescale-credentials + defaultMode: 0400 diff --git a/k8s/logging-and-monitoring/analyze.py b/k8s/logging-and-monitoring/analyze.py new file mode 100644 index 0000000..61ff2dc --- /dev/null +++ b/k8s/logging-and-monitoring/analyze.py @@ -0,0 +1,82 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +""" +A little Python script to do some analysis of the NGINX logs. +To get the relevant NGINX logs: +1. Go to the OMS Portal +2. Create a new Log Search +3. Use a search string such as: + +Type=ContainerLog Image="bigchaindb/nginx_3scale:1.3" GET NOT("Go-http-client") NOT(runscope) + +(This gets all logs from the NGINX container, only those with the word "GET", +excluding those with the string "Go-http-client" [internal Kubernetes traffic], +excluding those with the string "runscope" [Runscope tests].) + +4. In the left sidebar, at the top, use the dropdown menu to select the time range, +e.g. "Data based on last 7 days". Pay attention to the number of results and +the time series chart in the left sidebar. Are there any spikes? +5. Export the search results. A CSV file will be saved on your local machine. +6. $ python3 analyze.py logs.csv + +Thanks to https://gist.github.com/hreeder/f1ffe1408d296ce0591d +""" + +import sys +import csv +import re +from dateutil.parser import parse + + +lineformat = re.compile(r'(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - ' + r'\[(?P\d{2}\/[a-z]{3}\/\d{4}:\d{2}:\d{2}:\d{2} ' + r'(\+|\-)\d{4})\] ((\"(GET|POST) )(?P.+)(http\/1\.1")) ' + r'(?P\d{3}) ' + r'(?P\d+) ' + r'(["](?P(\-)|(.+))["]) ' + r'(["](?P.+)["])', + re.IGNORECASE) + +filepath = sys.argv[1] + +logline_list = [] +with open(filepath) as csvfile: + csvreader = csv.reader(csvfile, delimiter=',') + for row in csvreader: + if row and (row[8] != 'LogEntry'): + # because the first line is just the column headers, such as 'LogEntry' + logline = row[8] + print(logline + '\n') + logline_data = re.search(lineformat, logline) + if logline_data: + logline_dict = logline_data.groupdict() + logline_list.append(logline_dict) + # so logline_list is a list of dicts + # print('{}'.format(logline_dict)) + +# Analysis + +total_bytes_sent = 0 +tstamp_list = [] + +for lldict in logline_list: + total_bytes_sent += int(lldict['bytessent']) + dt = lldict['dateandtime'] + # https://tinyurl.com/lqjnhot + dtime = parse(dt[:11] + " " + dt[12:]) + tstamp_list.append(dtime.timestamp()) + +print('Number of log lines seen: {}'.format(len(logline_list))) + +# Time range +trange_sec = max(tstamp_list) - min(tstamp_list) +trange_days = trange_sec / 60.0 / 60.0 / 24.0 +print('Time range seen (days): {}'.format(trange_days)) + +print('Total bytes sent: {}'.format(total_bytes_sent)) + +print('Average bytes sent per day (out via GET): {}'. + format(total_bytes_sent / trange_days)) diff --git a/k8s/logging-and-monitoring/log_analytics_oms.json b/k8s/logging-and-monitoring/log_analytics_oms.json new file mode 100644 index 0000000..44b9d05 --- /dev/null +++ b/k8s/logging-and-monitoring/log_analytics_oms.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sku": { + "type": "String" + }, + "workspaceName": { + "type": "String" + }, + "solutionType": { + "type": "String" + } + }, + "resources": [ + { + "apiVersion": "2015-03-20", + "type": "Microsoft.OperationalInsights/workspaces", + "name": "[parameters('workspaceName')]", + "location": "[resourceGroup().location]", + "properties": { + "sku": { + "name": "[parameters('sku')]" + } + }, + "resources": [ + { + "apiVersion": "2015-11-01-preview", + "location": "[resourceGroup().location]", + "name": "[Concat(parameters('solutionType'), '(', parameters('workspaceName'), ')')]", + "type": "Microsoft.OperationsManagement/solutions", + "id": "[Concat(resourceGroup().id, '/providers/Microsoft.OperationsManagement/solutions/', parameters('solutionType'), '(', parameters('workspaceName'), ')')]", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]" + ], + "properties": { + "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]" + }, + "plan": { + "publisher": "Microsoft", + "product": "[Concat('OMSGallery/', parameters('solutionType'))]", + "name": "[Concat(parameters('solutionType'), '(', parameters('workspaceName'), ')')]", + "promotionCode": "" + } + } + ] + } + ] +} diff --git a/k8s/logging-and-monitoring/log_analytics_oms.parameters.json b/k8s/logging-and-monitoring/log_analytics_oms.parameters.json new file mode 100644 index 0000000..895cbb1 --- /dev/null +++ b/k8s/logging-and-monitoring/log_analytics_oms.parameters.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sku": { + "value": "Free" + }, + "workspaceName": { + "value": "rg-abc-logs" + }, + "solutionType": { + "value": "Containers" + } + } +} diff --git a/k8s/logging-and-monitoring/oms-daemonset.yaml b/k8s/logging-and-monitoring/oms-daemonset.yaml new file mode 100644 index 0000000..77eff46 --- /dev/null +++ b/k8s/logging-and-monitoring/oms-daemonset.yaml @@ -0,0 +1,35 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: omsagent +spec: + template: + metadata: + labels: + app: omsagent + spec: + containers: + - env: + - name: WSID + value: + - name: KEY + value: + image: microsoft/oms + name: omsagent + ports: + - containerPort: 25225 + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock diff --git a/k8s/mongodb-monitoring-agent/container/Dockerfile b/k8s/mongodb-monitoring-agent/container/Dockerfile new file mode 100644 index 0000000..d958d4b --- /dev/null +++ b/k8s/mongodb-monitoring-agent/container/Dockerfile @@ -0,0 +1,58 @@ +# Dockerfile for MongoDB Monitoring Agent +# Use it to create bigchaindb/mongodb-monitoring-agent +# on Docker Hub. + +# "Never install the Monitoring Agent on the same server as a data bearing mongod instance." +# More help: +# https://docs.cloudmanager.mongodb.com/tutorial/install-monitoring-agent-with-deb-package/ + +FROM ubuntu:xenial +LABEL maintainer "contact@ipdb.global" +# Using ARG, one can set DEBIAN_FRONTEND=noninteractive and others +# just for the duration of the build: +ARG DEBIAN_FRONTEND=noninteractive +ARG DEB_FILE=mongodb-mms-monitoring-agent_latest_amd64.ubuntu1604.deb +ARG FILE_URL="https://cloud.mongodb.com/download/agent/monitoring/"$DEB_FILE + +# Download the Monitoring Agent as a .deb package and install it +WORKDIR / +RUN apt update \ + && apt -y upgrade \ + && apt -y install --no-install-recommends \ + curl \ + ca-certificates \ + logrotate \ + libsasl2-2 \ + && curl -OL $FILE_URL \ + && dpkg -i $DEB_FILE \ + && rm -f $DEB_FILE \ + && apt -y purge curl \ + && apt -y autoremove \ + && apt clean + +# The above installation puts a default config file in +# /etc/mongodb-mms/monitoring-agent.config +# It should contain a line like: "mmsApiKey=" +# i.e. with no value specified. +# We need to set that value to the "agent API key" value from Cloud Manager, +# but of course that value varies from user to user, +# so we can't hard-code it into the Docker image. + +# Kubernetes can set an MMS_API_KEY environment variable +# in the container +# (including from Secrets or ConfigMaps) +# An entrypoint bash script can then use the value of MMS_API_KEY +# to write the mmsApiKey value in the config file +# /etc/mongodb-mms/monitoring-agent.config +# before running the MongoDB Monitoring Agent. + +# The MongoDB Monitoring Agent has other +# config settings besides mmsApiKey, +# but it's the only one that *must* be set. See: +# https://docs.cloudmanager.mongodb.com/reference/monitoring-agent/ + +COPY mongodb_mon_agent_entrypoint.bash / +RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/ +VOLUME /etc/mongod/ssl /etc/mongod/ca +USER mongodb-mms-agent +ENTRYPOINT ["/mongodb_mon_agent_entrypoint.bash"] \ No newline at end of file diff --git a/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash b/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash new file mode 100755 index 0000000..98fd135 --- /dev/null +++ b/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/mongodb-monitoring-agent:2.2.2 . + +docker push bigchaindb/mongodb-monitoring-agent:2.2.2 diff --git a/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash b/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash new file mode 100755 index 0000000..758cfb4 --- /dev/null +++ b/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -euo pipefail +# -e Abort at the first failed line (i.e. if exit status is not 0) +# -u Abort when undefined variable is used +# -o pipefail (Bash-only) Piped commands return the status +# of the last failed command, rather than the status of the last command + +MONGODB_MON_CONF_FILE=/etc/mongodb-mms/monitoring-agent.config + +mms_api_keyfile_path=`printenv MMS_API_KEYFILE_PATH` +mms_groupid_keyfile_path=`printenv MMS_GROUPID_KEYFILE_PATH` +ca_crt_path=`printenv CA_CRT_PATH` +monitoring_pem_path=`printenv MONITORING_PEM_PATH` + +if [[ -z "${mms_api_keyfile_path:?MMS_API_KEYFILE_PATH not specified. Exiting!}" || \ + -z "${ca_crt_path:?CA_CRT_PATH not specified. Exiting!}" || \ + -z "${monitoring_pem_path:?MONITORING_PEM_PATH not specified. Exiting!}" || \ + -z "${mms_groupid_keyfile_path:?MMS_GROUPID_KEYFILE_PATH not specified. Exiting!}" ]];then + exit 1 +else + echo MMS_API_KEYFILE_PATH="$mms_api_keyfile_path" + echo MMS_GROUPID_KEYFILE_PATH="$mms_groupid_keyfile_path" + echo CA_CRT_PATH="$ca_crt_path" + echo MONITORING_PEM_PATH="$monitoring_pem_path" +fi + +# Delete the line containing "mmsApiKey" and the line containing "mmsGroupId" +# in the MongoDB Monitoring Agent config file +# /etc/mongodb-mms/monitoring-agent.config +sed -i '/mmsApiKey/d' $MONGODB_MON_CONF_FILE +sed -i '/mmsGroupId/d' $MONGODB_MON_CONF_FILE + +# Get the api key from file +mms_api_key=`cat ${mms_api_keyfile_path}` +mms_groupid_key=`cat ${mms_groupid_keyfile_path}` + +# Append a new line of the form +# mmsApiKey=value_of_MMS_API_KEY +echo "mmsApiKey="${mms_api_key} >> ${MONGODB_MON_CONF_FILE} +echo "mmsGroupId="${mms_groupid_key} >> ${MONGODB_MON_CONF_FILE} + +# Append SSL settings to the config file +echo "useSslForAllConnections=true" >> ${MONGODB_MON_CONF_FILE} +echo "sslRequireValidServerCertificates=true" >> ${MONGODB_MON_CONF_FILE} +echo "sslTrustedServerCertificates="${ca_crt_path} >> ${MONGODB_MON_CONF_FILE} +echo "sslClientCertificate="${monitoring_pem_path} >> ${MONGODB_MON_CONF_FILE} +echo "#sslClientCertificatePassword=" >> ${MONGODB_MON_CONF_FILE} + +# start mdb monitoring agent +echo "INFO: starting mdb monitor..." +exec mongodb-mms-monitoring-agent \ + --conf $MONGODB_MON_CONF_FILE \ + --loglevel debug diff --git a/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml b/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml new file mode 100644 index 0000000..42fdd94 --- /dev/null +++ b/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml @@ -0,0 +1,70 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +############################################################ +# This config file defines a k8s Deployment for the # +# bigchaindb/mongodb-monitoring-agent Docker image # +# # +# It connects to a MongoDB instance in a separate pod, # +# all remote MongoDB instances in the cluster, # +# and also to MongoDB Cloud Manager (an external service). # +# Notes: # +# MongoDB agents connect to Cloud Manager on port 443. # +############################################################ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mdb-mon-instance-0-dep +spec: + replicas: 1 + template: + metadata: + name: mdb-mon-instance-0-dep + labels: + app: mdb-mon-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mdb-mon + image: bigchaindb/mongodb-monitoring-agent:2.2.2 + imagePullPolicy: IfNotPresent + env: + - name: MMS_API_KEYFILE_PATH + value: /etc/mongod/cloud/agent-api-key + - name: MMS_GROUPID_KEYFILE_PATH + value: /etc/mongod/cloud/group-id + - name: CA_CRT_PATH + value: /etc/mongod/ca/ca.pem + - name: MONITORING_PEM_PATH + value: /etc/mongod/ssl/mdb-mon-instance.pem + resources: + limits: + cpu: 200m + memory: 768Mi + volumeMounts: + - name: mdb-mon-certs + mountPath: /etc/mongod/ssl/ + readOnly: true + - name: ca-auth + mountPath: /etc/mongod/ca/ + readOnly: true + - name: cloud-manager-credentials + mountPath: /etc/mongod/cloud/ + readOnly: true + restartPolicy: Always + volumes: + - name: mdb-mon-certs + secret: + secretName: mdb-mon-certs + defaultMode: 0404 + - name: ca-auth + secret: + secretName: ca-auth + defaultMode: 0404 + - name: cloud-manager-credentials + secret: + secretName: cloud-manager-credentials + defaultMode: 0404 diff --git a/k8s/mongodb/configure_mdb.sh b/k8s/mongodb/configure_mdb.sh new file mode 100755 index 0000000..2449f13 --- /dev/null +++ b/k8s/mongodb/configure_mdb.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +# Checking if kubectl exists +command -v kubectl > /dev/null +if [ $? -eq 0 ] +then + echo "kubectl already installed!" +else + echo "Please install kubectl!! https://kubernetes.io/docs/tasks/tools/install-kubectl/" + exit 1 +fi + +MONGODB_INSTANCE_NAME="mdb-instance-0" + +if [[ -n "$MONGODB_INSTANCE_NAME" ]]; then + /usr/local/bin/kubectl exec -it "${MONGODB_INSTANCE_NAME}"\-ss\-0 -- bash -c "/usr/bin/mongo --host localhost --port \$(printenv MONGODB_PORT) --ssl --sslCAFile /etc/mongod/ca/ca.pem --sslPEMKeyFile /etc/mongod/ssl/mdb-instance.pem < /configure_mdb_users.js" +else + echo "Skipping configuration, because relevant files don't exist!!!" +fi diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile new file mode 100644 index 0000000..a2ade8f --- /dev/null +++ b/k8s/mongodb/container/Dockerfile @@ -0,0 +1,13 @@ +FROM mongo:3.6 +LABEL maintainer "contact@ipdb.global" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY mongod.conf.template /etc/mongod.conf +COPY configure_mdb_users.template.js /configure_mdb_users.js +COPY mongod_entrypoint.bash / +VOLUME /data/db /data/configdb /etc/mongod/ssl /etc/mongod/ca +EXPOSE 27017 +ENTRYPOINT ["/mongod_entrypoint.bash"] diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md new file mode 100644 index 0000000..4456876 --- /dev/null +++ b/k8s/mongodb/container/README.md @@ -0,0 +1,35 @@ + + +## Custom MongoDB container for Planetmint Backend + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + + +### Step 2: Run the Container + +``` +docker run \ + --cap-add=FOWNER \ + --name=mdb1 \ + --publish=: \ + --rm=true \ + --volume=:/data/db \ + --volume=:/data/configdb \ + --volume=:/mongo-ssl:ro \ + bigchaindb/mongodb: \ + --mongodb-port \ + --mongodb-key-file-path /mongo-ssl/.pem \ + --mongodb-ca-file-path /mongo-ssl/.crt \ + --mongodb-crl-file-path /mongo-ssl/.pem \ + --mongodb-fqdn \ + --mongodb-ip +``` diff --git a/k8s/mongodb/container/configure_mdb_users.template.js b/k8s/mongodb/container/configure_mdb_users.template.js new file mode 100644 index 0000000..4cf2021 --- /dev/null +++ b/k8s/mongodb/container/configure_mdb_users.template.js @@ -0,0 +1,58 @@ +// Copyright © 2020 Interplanetary Database Association e.V., +// Planetmint and IPDB software contributors. +// SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +// Code is Apache-2.0 and docs are CC-BY-4.0 + +var configure_adminUser = CONFIGURE_ADMIN_USER; +var configure_bdbUser = CONFIGURE_BDB_USER; +var configure_mdbMonUser = CONFIGURE_MDB_MON_USER; +db = db.getSiblingDB("admin"); + +if (configure_adminUser) { + db.createUser({ + user: "MONGODB_ADMIN_USERNAME", + pwd: "MONGODB_ADMIN_PASSWORD", + roles: [{ + role: "userAdminAnyDatabase", + db: "admin" + }, + { + role: "clusterManager", + db: "admin" + } + ] + }); +} +if (configure_adminUser && configure_bdbUser) { + db.auth("MONGODB_ADMIN_USERNAME", "MONGODB_ADMIN_PASSWORD"); + db.getSiblingDB("$external").runCommand({ + createUser: 'BDB_USERNAME', + writeConcern: { + w: 'majority', + wtimeout: 5000 + }, + roles: [{ + role: 'clusterAdmin', + db: 'admin' + }, + { + role: 'readWriteAnyDatabase', + db: 'admin' + } + ] + }); +} +if (configure_adminUser && configure_mdbMonUser) { + db.auth("MONGODB_ADMIN_USERNAME", "MONGODB_ADMIN_PASSWORD"); + db.getSiblingDB("$external").runCommand({ + createUser: 'MDB_MON_USERNAME', + writeConcern: { + w: 'majority', + wtimeout: 5000 + }, + roles: [{ + role: 'clusterMonitor', + db: 'admin' + }] + }); +} \ No newline at end of file diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash new file mode 100755 index 0000000..283afda --- /dev/null +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/localmongodb:2.2.2 . +docker push bigchaindb/localmongodb:2.2.2 diff --git a/k8s/mongodb/container/mongod.conf.template b/k8s/mongodb/container/mongod.conf.template new file mode 100644 index 0000000..c0a2d28 --- /dev/null +++ b/k8s/mongodb/container/mongod.conf.template @@ -0,0 +1,95 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# where to write logging data. +systemLog: + verbosity: 0 + # traceAllExceptions: true + timeStampFormat: iso8601-utc + component: + accessControl: + verbosity: 0 + command: + verbosity: 0 + control: + verbosity: 0 + ftdc: + verbosity: 0 + geo: + verbosity: 0 + index: + verbosity: 0 + network: + verbosity: 0 + query: + verbosity: 0 + sharding: + verbosity: 0 + storage: + verbosity: 0 + journal: + verbosity: 0 + write: + verbosity: 0 + +processManagement: + fork: false + pidFilePath: /tmp/mongod.pid + +net: + port: MONGODB_PORT + bindIp: 0.0.0.0 + maxIncomingConnections: 8192 + wireObjectCheck: false + unixDomainSocket: + enabled: false + pathPrefix: /tmp + filePermissions: 0700 + http: + enabled: false + compression: + compressors: snappy + ssl: + mode: requireSSL + PEMKeyFile: MONGODB_KEY_FILE_PATH + #PEMKeyPassword: MONGODB_KEY_FILE_PASSWORD + CAFile: MONGODB_CA_FILE_PATH + CRLFile: MONGODB_CRL_FILE_PATH + + #allowConnectionsWithoutCertificates: false + #allowInvalidHostnames: false + #weakCertificateValidation: false + #allowInvalidCertificates: false + +security: + authorization: enabled + clusterAuthMode: x509 + +setParameter: + enableLocalhostAuthBypass: true + #notablescan: 1 + logUserIds: 1 + authenticationMechanisms: MONGODB-X509,SCRAM-SHA-1 + +storage: + dbPath: /data/db/main + indexBuildRetry: true + journal: + enabled: true + commitIntervalMs: 100 + directoryPerDB: true + engine: wiredTiger + wiredTiger: + engineConfig: + journalCompressor: snappy + configString: cache_size=STORAGE_ENGINE_CACHE_SIZE + collectionConfig: + blockCompressor: snappy + indexConfig: + prefixCompression: true # TODO false may affect performance? + +operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash new file mode 100755 index 0000000..2facb6a --- /dev/null +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -0,0 +1,147 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +MONGODB_PORT="" +MONGODB_KEY_FILE_PATH="" +MONGODB_CA_FILE_PATH="" +MONGODB_CRL_FILE_PATH="" +MONGODB_FQDN="" +MONGODB_IP="" + +# vars for MongoDB configuration +configure_mongo=true +MONGODB_CREDENTIALS_DIR=/tmp/mongodb +mongodb_admin_password="" +mongodb_admin_username=`printenv MONGODB_ADMIN_USERNAME || true` +bdb_username=`printenv BDB_USERNAME || true` +mdb_mon_username=`printenv MDB_MON_USERNAME || true` + +while [[ $# -gt 1 ]]; do + arg="$1" + case $arg in + --mongodb-port) + MONGODB_PORT="$2" + shift + ;; + --mongodb-key-file-path) + MONGODB_KEY_FILE_PATH="$2" + shift + ;; + --mongodb-ca-file-path) + MONGODB_CA_FILE_PATH="$2" + shift + ;; + --mongodb-crl-file-path) + MONGODB_CRL_FILE_PATH="$2" + shift + ;; + --mongodb-fqdn) + MONGODB_FQDN="$2" + shift + ;; + --mongodb-ip) + MONGODB_IP="$2" + shift + ;; + --storage-engine-cache-size) + STORAGE_ENGINE_CACHE_SIZE="$2" + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift +done + +# sanity checks +if [[ -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \ + -z "${MONGODB_FQDN:?MONGODB_FQDN not specified. Exiting!}" || \ + -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ + -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ + -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ + -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then + # Not handling the STORAGE_ENGINE_CACHE_SIZE because + # it is optional. If not specified the default cache + # size is: max((50% RAM - 1GB), 256MB) + exit 1 +else + echo MONGODB_PORT="$MONGODB_PORT" + echo MONGODB_FQDN="$MONGODB_FQDN" + echo MONGODB_IP="$MONGODB_IP" + echo MONGODB_KEY_FILE_PATH="$MONGODB_KEY_FILE_PATH" + echo MONGODB_CA_FILE_PATH="$MONGODB_CA_FILE_PATH" + echo MONGODB_CRL_FILE_PATH="$MONGODB_CRL_FILE_PATH" + echo STORAGE_ENGINE_CACHE_SIZE="$STORAGE_ENGINE_CACHE_SIZE" +fi + +MONGODB_CONF_FILE_PATH=/etc/mongod.conf +HOSTS_FILE_PATH=/etc/hosts +MONGODB_CONFIGURE_USERS_PATH=/configure_mdb_users.js + +# configure the mongod.conf file +sed -i "s|MONGODB_PORT|${MONGODB_PORT}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|MONGODB_KEY_FILE_PATH|${MONGODB_KEY_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|MONGODB_CA_FILE_PATH|${MONGODB_CA_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|MONGODB_CRL_FILE_PATH|${MONGODB_CRL_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +if [ ! -z "$STORAGE_ENGINE_CACHE_SIZE" ]; then + if [[ "$STORAGE_ENGINE_CACHE_SIZE" =~ ^[0-9]+(G|M|T)B$ ]]; then + sed -i.bk "s|STORAGE_ENGINE_CACHE_SIZE|${STORAGE_ENGINE_CACHE_SIZE}|g" ${MONGODB_CONF_FILE_PATH} + else + echo "Invalid Value for storage engine cache size $STORAGE_ENGINE_CACHE_SIZE" + exit 1 + fi +else + sed -i.bk "/cache_size=/d" ${MONGODB_CONF_FILE_PATH} +fi + +if [ -f ${MONGODB_CREDENTIALS_DIR}/mdb-admin-password ]; then + mongodb_admin_password=`cat ${MONGODB_CREDENTIALS_DIR}/mdb-admin-password` +fi + +# Only configure if all variables are set +if [[ -n "${mongodb_admin_username}" && \ + -n "${mongodb_admin_password}" ]]; then + sed -i "s|MONGODB_ADMIN_USERNAME|${mongodb_admin_username}|g" ${MONGODB_CONFIGURE_USERS_PATH} + sed -i "s|MONGODB_ADMIN_PASSWORD|${mongodb_admin_password}|g" ${MONGODB_CONFIGURE_USERS_PATH} + sed -i "s|CONFIGURE_ADMIN_USER|true|g" ${MONGODB_CONFIGURE_USERS_PATH} +else + sed -i "s|CONFIGURE_ADMIN_USER|false|g" ${MONGODB_CONFIGURE_USERS_PATH} +fi + +if [[ -n "${bdb_username}" ]]; then + sed -i "s|BDB_USERNAME|${bdb_username}|g" ${MONGODB_CONFIGURE_USERS_PATH} + sed -i "s|CONFIGURE_BDB_USER|true|g" ${MONGODB_CONFIGURE_USERS_PATH} +else + sed -i "s|CONFIGURE_BDB_USER|false|g" ${MONGODB_CONFIGURE_USERS_PATH} +fi + +if [[ -n "${mdb_mon_username}" ]]; then + sed -i "s|MDB_MON_USERNAME|${mdb_mon_username}|g" ${MONGODB_CONFIGURE_USERS_PATH} + sed -i "s|CONFIGURE_MDB_MON_USER|true|g" ${MONGODB_CONFIGURE_USERS_PATH} +else + sed -i "s|CONFIGURE_MDB_MON_USER|false|g" ${MONGODB_CONFIGURE_USERS_PATH} +fi + +# add the hostname and ip to hosts file +echo "${MONGODB_IP} ${MONGODB_FQDN}" >> $HOSTS_FILE_PATH + +# create the directory if it does not exist, where MongoDB can store the data +# and config files; this assumes that the data directory is mounted at +# /data/db/main and the config directory is mounted at /data/configdb +mkdir -p /data/db/main /data/configdb/main + +# start mongod +echo "INFO: starting mongod..." + +# TODO Uncomment the first exec command and use it instead of the second one +# after https://github.com/docker-library/mongo/issues/172 is resolved. Check +# for other bugs too. +#exec /entrypoint.sh mongod --config ${MONGODB_CONF_FILE_PATH} +exec /usr/bin/mongod --config ${MONGODB_CONF_FILE_PATH} diff --git a/k8s/mongodb/mongo-ext-conn-svc.yaml b/k8s/mongodb/mongo-ext-conn-svc.yaml new file mode 100644 index 0000000..6d7ee34 --- /dev/null +++ b/k8s/mongodb/mongo-ext-conn-svc.yaml @@ -0,0 +1,18 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + # Name of mongodb instance you are trying to connect to + # e.g. mdb-instance-0 + name: "" + namespace: default +spec: + ports: + - port: "" + type: ExternalName + # FQDN of remote cluster/NGINX instance + externalName: "" \ No newline at end of file diff --git a/k8s/mongodb/mongo-pv.yaml b/k8s/mongodb/mongo-pv.yaml new file mode 100644 index 0000000..73b8c7f --- /dev/null +++ b/k8s/mongodb/mongo-pv.yaml @@ -0,0 +1,46 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +############################################################# +# This YAML section desribes a k8s PV for mongodb dbPath # +############################################################# +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-mongo-db +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 50Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: slow-db +--- +############################################################# +# This YAML section desribes a k8s PV for mongodb configDB # +############################################################# +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-mongdo-configdb +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 2Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: slow-configdb diff --git a/k8s/mongodb/mongo-pvc.yaml b/k8s/mongodb/mongo-pvc.yaml new file mode 100644 index 0000000..a33dd03 --- /dev/null +++ b/k8s/mongodb/mongo-pvc.yaml @@ -0,0 +1,36 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +########################################################### +# This section file desribes a k8s pvc for mongodb dbPath # +########################################################### +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mongo-db-claim + annotations: + volume.beta.kubernetes.io/storage-class: slow-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +############################################################# +# This YAML section desribes a k8s pvc for mongodb configDB # +############################################################# +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mongo-configdb-claim + annotations: + volume.beta.kubernetes.io/storage-class: slow-configdb +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/k8s/mongodb/mongo-sc.yaml b/k8s/mongodb/mongo-sc.yaml new file mode 100644 index 0000000..83047ed --- /dev/null +++ b/k8s/mongodb/mongo-sc.yaml @@ -0,0 +1,36 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +#################################################################### +# This YAML section desribes a StorageClass for the mongodb dbPath # +#################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow-db +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] + location: + # If you have created a different storage account e.g. for Premium Storage + storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed +--- +###################################################################### +# This YAML section desribes a StorageClass for the mongodb configDB # +###################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow-configdb +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] + location: + # If you have created a different storage account e.g. for Premium Storage + storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml new file mode 100644 index 0000000..f61f3e6 --- /dev/null +++ b/k8s/mongodb/mongo-ss.yaml @@ -0,0 +1,134 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +######################################################################## +# This YAML file desribes a StatefulSet with a service for running and # +# exposing a MongoDB instance. # +# It depends on the configdb and db k8s pvc. # +######################################################################## + +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: mdb-instance-0-ss + namespace: default +spec: + serviceName: mdb-instance-0 + replicas: 1 + template: + metadata: + name: mdb-instance-0-ss + labels: + app: mdb-instance-0-ss + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongodb + image: bigchaindb/localmongodb:2.2.2 + imagePullPolicy: Always + env: + - name: MONGODB_FQDN + valueFrom: + configMapKeyRef: + name: vars + key: mdb-instance-name + - name: MONGODB_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MONGODB_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: STORAGE_ENGINE_CACHE_SIZE + valueFrom: + configMapKeyRef: + name: vars + key: storage-engine-cache-size + - name: MONGODB_ADMIN_USERNAME + valueFrom: + configMapKeyRef: + name: mdb-config + key: mdb-admin-username + - name: BDB_USERNAME + valueFrom: + configMapKeyRef: + name: bdb-config + key: bdb-user + - name: MDB_MON_USERNAME + valueFrom: + configMapKeyRef: + name: mdb-config + key: mdb-mon-user + args: + - --mongodb-port + - $(MONGODB_PORT) + - --mongodb-key-file-path + - /etc/mongod/ssl/mdb-instance.pem + - --mongodb-ca-file-path + - /etc/mongod/ca/ca.pem + - --mongodb-crl-file-path + - /etc/mongod/ca/crl.pem + - --mongodb-fqdn + - $(MONGODB_FQDN) + - --mongodb-ip + - $(MONGODB_POD_IP) + - --storage-engine-cache-size + - $(STORAGE_ENGINE_CACHE_SIZE) + securityContext: + capabilities: + add: + - FOWNER + ports: + - containerPort: 27017 + protocol: TCP + name: mdb-api-port + volumeMounts: + - name: mdb-db + mountPath: /data/db + - name: mdb-configdb + mountPath: /data/configdb + - name: mdb-certs + mountPath: /etc/mongod/ssl/ + readOnly: true + - name: ca-auth + mountPath: /etc/mongod/ca/ + readOnly: true + - name: mdb-config + mountPath: /tmp/mongodb + readOnly: true + resources: + limits: + cpu: 200m + memory: 5G + livenessProbe: + tcpSocket: + port: mdb-api-port + initialDelaySeconds: 15 + successThreshold: 1 + failureThreshold: 3 + periodSeconds: 15 + timeoutSeconds: 10 + restartPolicy: Always + volumes: + - name: mdb-db + persistentVolumeClaim: + claimName: mongo-db-claim + - name: mdb-configdb + persistentVolumeClaim: + claimName: mongo-configdb-claim + - name: mdb-certs + secret: + secretName: mdb-certs + defaultMode: 0400 + - name: ca-auth + secret: + secretName: ca-auth + defaultMode: 0400 + - name: mdb-config + secret: + secretName: mdb-config + defaultMode: 0400 diff --git a/k8s/mongodb/mongo-svc.yaml b/k8s/mongodb/mongo-svc.yaml new file mode 100644 index 0000000..fc1cb42 --- /dev/null +++ b/k8s/mongodb/mongo-svc.yaml @@ -0,0 +1,22 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: mdb-instance-0 + namespace: default + labels: + name: mdb-instance-0 +spec: + selector: + app: mdb-instance-0-ss + ports: + - port: 27017 + targetPort: 27017 + name: mdb-port + protocol: TCP + type: ClusterIP + clusterIP: None diff --git a/k8s/nginx-http/container/Dockerfile b/k8s/nginx-http/container/Dockerfile new file mode 100644 index 0000000..b97e7f9 --- /dev/null +++ b/k8s/nginx-http/container/Dockerfile @@ -0,0 +1,11 @@ +FROM nginx:stable +LABEL maintainer "contact@ipdb.global" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /etc/nginx/nginx.conf +COPY nginx_entrypoint.bash / +EXPOSE 80 27017 9986 26656 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-http/container/README.md b/k8s/nginx-http/container/README.md new file mode 100644 index 0000000..6f4e444 --- /dev/null +++ b/k8s/nginx-http/container/README.md @@ -0,0 +1,22 @@ + + +## Nginx container for Secure WebSocket Support + + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + + +### Note about testing Websocket connections: +You can test the WebSocket server by using +[wsc](https://www.npmjs.com/package/wsc) tool with a command like: + +`wsc -er ws://localhost:9985/api/v1/streams/valid_transactions`. diff --git a/k8s/nginx-http/container/docker_build_and_push.bash b/k8s/nginx-http/container/docker_build_and_push.bash new file mode 100755 index 0000000..dd4a07d --- /dev/null +++ b/k8s/nginx-http/container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/nginx_http:2.2.2 + +docker push bigchaindb/nginx_http:2.2.2 diff --git a/k8s/nginx-http/container/nginx.conf.template b/k8s/nginx-http/container/nginx.conf.template new file mode 100644 index 0000000..95e7cde --- /dev/null +++ b/k8s/nginx-http/container/nginx.conf.template @@ -0,0 +1,170 @@ +# Frontend API server that: +# 1. Forwards BDB HTTP & WS requests to BDB backend. +# 2. Forwards MDB TCP connections to MDB backend. +# 3. Does health check with LB. + +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /dev/stderr; + +events { + # Each worker handles up to 512 connections. Increase this for heavy + # workloads. + worker_connections 512; + accept_mutex on; + use epoll; +} + +http { + access_log /dev/stdout combined buffer=16k flush=5s; + + # Allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code to return to the client when throttling; + # 429 is for TooManyRequests, ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 10s; + client_header_timeout 10s; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + keepalive_timeout 60s; + + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + + # Frontend server for the external clients + server { + listen NODE_FRONTEND_PORT; + underscores_in_headers on; + + # Forward websockets to backend BDB at 9985. + location /api/v1/streams/valid_transactions { + proxy_pass http://$bdb_backend:PLANETMINT_WS_PORT; + proxy_read_timeout 600s; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + + # Forward other URL paths to backend BDB at 9984. + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + + # max client request body size: avg transaction size. + client_max_body_size 15k; + + # No auth for GETs, forward directly to BDB. + if ($request_method = GET) { + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + } + + # POST requests get forwarded to OpenResty instance. Enable CORS too. + if ($request_method = POST ) { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + } + + # OPTIONS requests handling for CORS. + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen HEALTH_CHECK_PORT; + + location = /health { + return 200; + } + } +} + +# NGINX stream block for TCP and UDP proxies. +stream { + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /dev/stdout bdb_log buffer=16k flush=5s; + + # Define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address. + limit_conn_zone $binary_remote_addr zone=two:10m; + + # Enable logging when connections are being throttled. + limit_conn_log_level notice; + + # Allow 16 connections from the same IP address. + limit_conn two 16; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + + # Server to forward connection to nginx instance hosting + # tendermint node public key. + server { + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $bdb_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; + tcp_nodelay on; + proxy_pass $bdb_backend:TM_P2P_PORT; + } +} + diff --git a/k8s/nginx-http/container/nginx_entrypoint.bash b/k8s/nginx-http/container/nginx_entrypoint.bash new file mode 100755 index 0000000..630c9da --- /dev/null +++ b/k8s/nginx-http/container/nginx_entrypoint.bash @@ -0,0 +1,79 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +# Cluster vars +node_frontend_port=`printenv NODE_FRONTEND_PORT` + + +# NGINX vars +dns_server=`printenv DNS_SERVER` +health_check_port=`printenv HEALTH_CHECK_PORT` + + +# MongoDB vars +mongo_frontend_port=`printenv MONGODB_FRONTEND_PORT` +mongo_backend_host=`printenv MONGODB_BACKEND_HOST` +mongo_backend_port=`printenv MONGODB_BACKEND_PORT` + + +# Planetmint vars +bdb_backend_host=`printenv PLANETMINT_BACKEND_HOST` +bdb_api_port=`printenv PLANETMINT_API_PORT` +bdb_ws_port=`printenv PLANETMINT_WS_PORT` + +# Tendermint vars +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` +tm_p2p_port=`printenv TM_P2P_PORT` + + +# sanity check +if [[ -z "${node_frontend_port:?NODE_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_frontend_port:?MONGODB_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_backend_host:?MONGODB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${mongo_backend_port:?MONGODB_BACKEND_PORT not specified. Exiting!}" || \ + -z "${bdb_backend_host:?PLANETMINT_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?PLANETMINT_API_PORT not specified. Exiting!}" || \ + -z "${bdb_ws_port:?PLANETMINT_WS_PORT not specified. Exiting!}" || \ + -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then + exit 1 +else + echo NODE_FRONTEND_PORT="$node_frontend_port" + echo DNS_SERVER="$dns_server" + echo HEALTH_CHECK_PORT="$health_check_port" + echo MONGODB_FRONTEND_PORT="$mongo_frontend_port" + echo MONGODB_BACKEND_HOST="$mongo_backend_host" + echo MONGODB_BACKEND_PORT="$mongo_backend_port" + echo PLANETMINT_BACKEND_HOST="$bdb_backend_host" + echo PLANETMINT_API_PORT="$bdb_api_port" + echo PLANETMINT_WS_PORT="$bdb_ws_port" + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" + echo TM_P2P_PORT="$tm_p2p_port" +fi + +NGINX_CONF_FILE=/etc/nginx/nginx.conf + +# configure the nginx.conf file with env variables +sed -i "s|NODE_FRONTEND_PORT|${node_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_FRONTEND_PORT|${mongo_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c /etc/nginx/nginx.conf + diff --git a/k8s/nginx-http/nginx-http-dep.yaml b/k8s/nginx-http/nginx-http-dep.yaml new file mode 100644 index 0000000..f7b906d --- /dev/null +++ b/k8s/nginx-http/nginx-http-dep.yaml @@ -0,0 +1,102 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: ngx-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: bigchaindb/nginx_http:2.2.2 + imagePullPolicy: Always + env: + - name: NODE_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: node-frontend-port + - name: HEALTH_CHECK_PORT + valueFrom: + configMapKeyRef: + name: vars + key: node-health-check-port + - name: DNS_SERVER + valueFrom: + configMapKeyRef: + name: vars + key: node-dns-server-ip + - name: MONGODB_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: MONGODB_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-mdb-instance-name + - name: MONGODB_BACKEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: PLANETMINT_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-bdb-instance-name + - name: PLANETMINT_API_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-api-port + - name: PLANETMINT_WS_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-port + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-pub-key-access + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-p2p-port + ports: + - containerPort: 8888 + protocol: TCP + name: ngx-health + - containerPort: 80 + protocol: TCP + - containerPort: 9986 + protocol: TCP + name: bdb-pub-key + - containerPort: 26656 + protocol: TCP + name: bdb-p2p-port + livenessProbe: + httpGet: + path: /health + port: ngx-health + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always diff --git a/k8s/nginx-http/nginx-http-svc.yaml b/k8s/nginx-http/nginx-http-svc.yaml new file mode 100644 index 0000000..fb1b745 --- /dev/null +++ b/k8s/nginx-http/nginx-http-svc.yaml @@ -0,0 +1,33 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: ngx-instance-0 + namespace: default + labels: + name: ngx-instance-0 + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-instance-0-dep + ports: + - port: "" + targetPort: "" + name: public-node-port + protocol: TCP + - port: "" + targetPort: "" + name: tm-pub-key-access + protocol: TCP + - port: "" + targetPort: "" + protocol: TCP + name: tm-p2p-port + type: LoadBalancer diff --git a/k8s/nginx-https-web-proxy/README.md b/k8s/nginx-https-web-proxy/README.md new file mode 100644 index 0000000..4dea96f --- /dev/null +++ b/k8s/nginx-https-web-proxy/README.md @@ -0,0 +1,40 @@ + + +## Deploying the Planetmint Web Proxy on a Kubernetes Cluster + + +### Configure the Web Proxy + +* Fill in the configuration details for the proxy in the + `nginx-https-web-proxy-conf.yaml` file. + +* Use the command below to create the appropriate ConfigMap and Secret: +``` +kubectl apply -f nginx-https-web-proxy-conf.yaml +``` + + +### Start the Kubernetes Service for Planetmint Web Proxy + +* Use the command below to start the Kubernetes Service: +``` +kubectl apply -f nginx-https-web-proxy-svc.yaml +``` + +* This will give you a public IP address tied to an Azure LB. + +* Map this to an available domain of your choice on the Azure portal (or use + any other DNS service provider!) + + +### Start the Kubernetes Deployment for Planetmint Web Proxy + +* Use the command below to start the Kubernetes Deployment: +``` +kubectl apply -f nginx-https-web-proxy-dep.yaml +``` diff --git a/k8s/nginx-https-web-proxy/container/Dockerfile b/k8s/nginx-https-web-proxy/container/Dockerfile new file mode 100644 index 0000000..7cb0868 --- /dev/null +++ b/k8s/nginx-https-web-proxy/container/Dockerfile @@ -0,0 +1,8 @@ +FROM openresty/openresty:alpine +RUN apk update \ + && apk upgrade \ + && apk add bash +COPY nginx.conf.template /etc/nginx/nginx.conf +COPY nginx_entrypoint.bash / +EXPOSE 443 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash b/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash new file mode 100755 index 0000000..caa4162 --- /dev/null +++ b/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/nginx-https-web-proxy:0.12 . + +docker push bigchaindb/nginx-https-web-proxy:0.12 diff --git a/k8s/nginx-https-web-proxy/container/nginx.conf.template b/k8s/nginx-https-web-proxy/container/nginx.conf.template new file mode 100644 index 0000000..6be443f --- /dev/null +++ b/k8s/nginx-https-web-proxy/container/nginx.conf.template @@ -0,0 +1,139 @@ +# Frontend Proxy server that: +# 1. Acts as the HTTPS proxy termination point. +# 2. Forwards BDB POST requests to OpenResty backend after appending the app_id +# and app_key headers. +# 3. Forwards BDB GET requests to BDB backend. +# 4. Does health check with LB. + +worker_processes 4; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /dev/stderr; + +events { + # Each worker handles up to 1024 connections. Increase this for heavy + # workloads. + worker_connections 1024; + accept_mutex on; + use epoll; +} + +http { + access_log /dev/stdout combined buffer=16k flush=5s; + + # Allow 2048 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=2048r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code that is returned to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 3072 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=3072 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 30s; + client_header_timeout 10s; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + keepalive_timeout 60s; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + map $remote_addr $openresty_backend { + default OPENRESTY_BACKEND_HOST; + } + + # Frontend server for the external clients; acts as HTTPS termination point. + server { + listen PROXY_FRONTEND_PORT ssl; + server_name "PROXY_FQDN"; + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/cert.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + underscores_in_headers on; + + # No websocket support for web proxy + location /api/v1/streams/valid_transactions { + return 403 'Websockets are not supported in the web proxy'; + } + + # Forward other URL paths as per business logic/use case to BDB or + # OpenResty instance. + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # max client request body size: avg transaction size. + client_max_body_size 15k; + + # Debug block for listing all the headers sent with the request + header_filter_by_lua_block { + local h = ngx.req.get_headers() + for k, v in pairs(h) do + ngx.log(ngx.ERR, "Header "..k..": "..v..";") + end + } + + # check if the request has the expected origin header + if ($http_origin !~ "PROXY_EXPECTED_ORIGIN_HEADER" ) { + return 403 'Unknown origin'; + } + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + add_header 'Referrer-Policy' "PROXY_REFERRER_POLICY"; + return 204; + } + + # check if the request originated from the required web page + # use referer header. + if ($http_referer !~ "PROXY_EXPECTED_REFERER_HEADER" ) { + return 403 'Unknown referer'; + } + + # No auth for GETs, forward directly to BDB. + if ($request_method = GET) { + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + } + + # POST requests get forwarded to OpenResty instance; set the correct + # headers accordingly + proxy_set_header app_id "PROXY_APP_ID"; + proxy_set_header app_key "PROXY_APP_KEY"; + if ($request_method = POST ) { + proxy_pass http://$openresty_backend:OPENRESTY_BACKEND_PORT; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen HEALTH_CHECK_PORT; + + location = /health { + return 200; + } + } +} + diff --git a/k8s/nginx-https-web-proxy/container/nginx_entrypoint.bash b/k8s/nginx-https-web-proxy/container/nginx_entrypoint.bash new file mode 100755 index 0000000..11da1ae --- /dev/null +++ b/k8s/nginx-https-web-proxy/container/nginx_entrypoint.bash @@ -0,0 +1,79 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +# Proxy vars +proxy_fqdn=`printenv PROXY_FQDN` +proxy_frontend_port=`printenv PROXY_FRONTEND_PORT` + +proxy_app_id_file=/etc/nginx/proxy/credentials/app_id +proxy_app_key_file=/etc/nginx/proxy/credentials/app_key +proxy_app_id=`cat ${proxy_app_id_file}` +proxy_app_key=`cat ${proxy_app_key_file}` + +proxy_expected_referer_header=`printenv PROXY_EXPECTED_REFERER_HEADER` +proxy_expected_origin_header=`printenv PROXY_EXPECTED_ORIGIN_HEADER` + +# OpenResty vars +openresty_backend_host=`printenv OPENRESTY_BACKEND_HOST` +openresty_backend_port=`printenv OPENRESTY_BACKEND_PORT` + +# NGINX vars +dns_server=`printenv DNS_SERVER` +health_check_port=`printenv HEALTH_CHECK_PORT` + +# Planetmint vars +bdb_backend_host=`printenv PLANETMINT_BACKEND_HOST` +bdb_api_port=`printenv PLANETMINT_API_PORT` + + +# sanity check +if [[ -z "${proxy_frontend_port:?PROXY_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${openresty_backend_port:?OPENRESTY_BACKEND_PORT not specified. Exiting!}" || \ + -z "${openresty_backend_host:?OPENRESTY_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_backend_host:?PLANETMINT_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?PLANETMINT_API_PORT not specified. Exiting!}" || \ + -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \ + -z "${proxy_app_id:?PROXY_APP_ID not specified. Exiting!}" || \ + -z "${proxy_app_key:?PROXY_APP_KEY not specified. Exiting!}" || \ + -z "${proxy_expected_referer_header:?PROXY_EXPECTED_REFERER_HEADER not specified. Exiting!}" || \ + -z "${proxy_expected_origin_header:?PROXY_EXPECTED_ORIGIN_HEADER not specified. Exiting!}" || \ + -z "${proxy_fqdn:?PROXY_FQDN not specified. Exiting!}" ]]; then + exit 1 +else + echo PROXY_FQDN="$proxy_fqdn" + echo PROXY_FRONTEND_PORT="$proxy_frontend_port" + echo PROXY_EXPECTED_REFERER_HEADER="$proxy_expected_referer_header" + echo PROXY_EXPECTED_ORIGIN_HEADER="$proxy_expected_origin_header" + echo DNS_SERVER="$dns_server" + echo HEALTH_CHECK_PORT="$health_check_port" + echo OPENRESTY_BACKEND_HOST="$openresty_backend_host" + echo OPENRESTY_BACKEND_PORT="$openresty_backend_port" + echo PLANETMINT_BACKEND_HOST="$bdb_backend_host" + echo PLANETMINT_API_PORT="$bdb_api_port" +fi + +NGINX_CONF_FILE=/etc/nginx/nginx.conf + +# configure the nginx.conf file with env variables +sed -i "s|PROXY_FQDN|${proxy_fqdn}|g" ${NGINX_CONF_FILE} +sed -i "s|PROXY_FRONTEND_PORT|${proxy_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|OPENRESTY_BACKEND_PORT|${openresty_backend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|OPENRESTY_BACKEND_HOST|${openresty_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|PROXY_APP_ID|${proxy_app_id}|g" ${NGINX_CONF_FILE} +sed -i "s|PROXY_APP_KEY|${proxy_app_key}|g" ${NGINX_CONF_FILE} +sed -i "s|PROXY_EXPECTED_REFERER_HEADER|${proxy_expected_referer_header}|g" ${NGINX_CONF_FILE} +sed -i "s|PROXY_EXPECTED_ORIGIN_HEADER|${proxy_expected_origin_header}|g" ${NGINX_CONF_FILE} + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c /etc/nginx/nginx.conf diff --git a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml new file mode 100644 index 0000000..1e6062e --- /dev/null +++ b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml @@ -0,0 +1,70 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# All secret data should be base64 encoded before embedding them in the Secret. +# Short strings can be encoded using, e.g. +# echo "secret string" | base64 -w 0 > secret.string.b64 +# Files (e.g. certificates) can be encoded using, e.g. +# cat cert.pem | base64 -w 0 > cert.pem.b64 +# then copy the contents of cert.pem.b64 (for example) below. +# Ref: https://kubernetes.io/docs/concepts/configuration/secret/ +# Unused values can be set to "" + +apiVersion: v1 +kind: Secret +metadata: + name: proxy-credentials + namespace: default +type: Opaque +data: + # app_id is the app id that the proxy adds to requests going to the backend + app_id: "" + # app_key is the app key that the proxy adds to requests going to the backend + app_key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: proxy-https-certs + namespace: default +type: Opaque +data: + # Base64-encoded HTTPS private key + cert.key: "" + # Base64-encoded HTTPS certificate chain + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert is from DigiCert, download "Best format for nginx". + cert.pem: " + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 256Mi + volumeMounts: + - name: proxy-https-certs + mountPath: /etc/nginx/ssl/ + readOnly: true + - name: proxy-credentials + mountPath: /etc/nginx/proxy/credentials/ + readOnly: true + restartPolicy: Always + volumes: + - name: proxy-https-certs + secret: + secretName: proxy-https-certs + defaultMode: 0400 + - name: proxy-credentials + secret: + secretName: proxy-credentials + defaultMode: 0400 diff --git a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-svc.yaml b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-svc.yaml new file mode 100644 index 0000000..819714a --- /dev/null +++ b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-svc.yaml @@ -0,0 +1,21 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: web-proxy-instance-0 + namespace: default + labels: + name: web-proxy-instance-0 +spec: + selector: + app: web-proxy-instance-0-dep + ports: + - port: + targetPort: + name: public-web-proxy-port + protocol: TCP + type: LoadBalancer diff --git a/k8s/nginx-https/container/Dockerfile b/k8s/nginx-https/container/Dockerfile new file mode 100644 index 0000000..0516add --- /dev/null +++ b/k8s/nginx-https/container/Dockerfile @@ -0,0 +1,12 @@ +FROM nginx:stable +LABEL maintainer "contact@ipdb.global" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.threescale.template /etc/nginx/nginx-threescale.conf +COPY nginx.conf.template /etc/nginx/nginx.conf +COPY nginx_entrypoint.bash / +EXPOSE 80 443 27017 9986 26656 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-https/container/README.md b/k8s/nginx-https/container/README.md new file mode 100644 index 0000000..53f7231 --- /dev/null +++ b/k8s/nginx-https/container/README.md @@ -0,0 +1,22 @@ + + +## Nginx container for Secure WebSocket Support + + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + + +### Note about testing Websocket connections: +You can test the WebSocket server by using +[wsc](https://www.npmjs.com/package/wsc) tool with a command like: + +`wsc -er wss://localhost:9985/api/v1/streams/valid_transactions`. diff --git a/k8s/nginx-https/container/docker_build_and_push.bash b/k8s/nginx-https/container/docker_build_and_push.bash new file mode 100755 index 0000000..ab89377 --- /dev/null +++ b/k8s/nginx-https/container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/nginx_https:2.2.2 . + +docker push bigchaindb/nginx_https:2.2.2 diff --git a/k8s/nginx-https/container/nginx.conf.template b/k8s/nginx-https/container/nginx.conf.template new file mode 100644 index 0000000..45361f1 --- /dev/null +++ b/k8s/nginx-https/container/nginx.conf.template @@ -0,0 +1,201 @@ +# Frontend API server that: +# 1. Acts as the HTTPS termination point. +# 2. Authorizes HTTP requests with secret token header +# and forwards to BDB backend. +# 3. Forwards BDB WS requests to BDB backend. +# 4. Does health check with LB. + +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /dev/stderr; + +events { + # Each worker handles up to 512 connections. Increase this for heavy + # workloads. + worker_connections 512; + accept_mutex on; + use epoll; +} + +http { + access_log /dev/stdout combined buffer=16k flush=5s; + + # Allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code that is returned to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 10s; + client_header_timeout 10s; + + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + keepalive_timeout 60s; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + + # Frontend server for the external clients; acts as HTTPS termination point. + server { + listen NODE_FRONTEND_PORT ssl; + server_name "NODE_FQDN"; + + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/cert.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + underscores_in_headers on; + + # Forward websockets directly to backend BDB. + location /api/v1/streams/valid_transactions { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://$bdb_backend:PLANETMINT_WS_PORT; + proxy_read_timeout 600s; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Forward other URL paths as per business logic/use case to BDB or + # OpenResty instance. + location / { + set $auth_check 1; #Flag to authorize POST requests + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # max client request body size: avg transaction size. + client_max_body_size 15k; + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-Secret-Access-Token,User-Agent'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + if ( $http_x_secret_access_token != "SECRET_ACCESS_TOKEN" ) { + set $auth_check 0; + } + if ($request_method = POST ) { + set $auth_check "${auth_check}1"; + } + + if ( $auth_check = "01" ) { + return 403; + } + + # No auth for GETs, forward directly to BDB. + if ($request_method = GET) { + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + } + + if ($request_method = POST ) { + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen HEALTH_CHECK_PORT; + + location = /health { + return 200; + } + } + + # Frontend server for the external clients; returns a pretty error message + # when an HTTP request is sent instead of HTTPS. + server { + listen 80; + server_name "NODE_FQDN"; + + location / { + add_header Upgrade "TLS/1.2, HTTP/1.1" always; + default_type text/plain; + return 426 'Consider using the HTTPS protocol next time!'; + } + } +} + +# NGINX stream block for TCP and UDP proxies. +stream { + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /dev/stdout bdb_log buffer=16k flush=5s; + + # Define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address. + limit_conn_zone $binary_remote_addr zone=two:10m; + + # Enable logging when connections are being throttled. + limit_conn_log_level notice; + + # Allow 256 connections from the same IP address. + limit_conn two 256; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + + # Server to forward connection to nginx instance hosting + # tendermint node public key. + server { + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $bdb_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; + tcp_nodelay on; + proxy_pass $bdb_backend:TM_P2P_PORT; + } +} + diff --git a/k8s/nginx-https/container/nginx.conf.threescale.template b/k8s/nginx-https/container/nginx.conf.threescale.template new file mode 100644 index 0000000..edf285a --- /dev/null +++ b/k8s/nginx-https/container/nginx.conf.threescale.template @@ -0,0 +1,198 @@ +# Frontend API server that: +# 1. Acts as the HTTPS termination point. +# 2. Forwards BDB HTTP requests to OpenResty backend. +# 3. Forwards BDB WS requests to BDB backend. +# 4. Does health check with LB. + +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /dev/stderr; + +events { + # Each worker handles up to 512 connections. Increase this for heavy + # workloads. + worker_connections 512; + accept_mutex on; + use epoll; +} + +http { + access_log /dev/stdout combined buffer=16k flush=5s; + + # Allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code that is returned to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 10s; + client_header_timeout 10s; + + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + keepalive_timeout 60s; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + map $remote_addr $openresty_backend { + default OPENRESTY_BACKEND_HOST; + } + + # Frontend server for the external clients; acts as HTTPS termination point. + server { + listen NODE_FRONTEND_PORT ssl; + server_name "NODE_FQDN"; + + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/cert.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + underscores_in_headers on; + + # Forward websockets directly to backend BDB. + location /api/v1/streams/valid_transactions { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://$bdb_backend:PLANETMINT_WS_PORT; + proxy_read_timeout 600s; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Forward other URL paths as per business logic/use case to BDB or + # OpenResty instance. + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # max client request body size: avg transaction size. + client_max_body_size 15k; + + # No auth for GETs, forward directly to BDB. + if ($request_method = GET) { + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + } + + # POST requests get forwarded to OpenResty instance. Enable CORS too. + if ($request_method = POST ) { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass http://$openresty_backend:OPENRESTY_BACKEND_PORT; + } + + # OPTIONS requests handling for CORS. + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen HEALTH_CHECK_PORT; + + location = /health { + return 200; + } + } + + # Frontend server for the external clients; returns a pretty error message + # when an HTTP request is sent instead of HTTPS. + server { + listen 80; + server_name "NODE_FQDN"; + + location / { + add_header Upgrade "TLS/1.2, HTTP/1.1" always; + default_type text/plain; + return 426 'Consider using the HTTPS protocol next time!'; + } + } +} + +# NGINX stream block for TCP and UDP proxies. +stream { + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /dev/stdout bdb_log buffer=16k flush=5s; + + # Define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address. + limit_conn_zone $binary_remote_addr zone=two:10m; + + # Enable logging when connections are being throttled. + limit_conn_log_level notice; + + # Allow 256 connections from the same IP address. + limit_conn two 256; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + + # Server to forward connection to nginx instance hosting + # tendermint node public key. + server { + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $bdb_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; + tcp_nodelay on; + proxy_pass $bdb_backend:TM_P2P_PORT; + } +} + diff --git a/k8s/nginx-https/container/nginx_entrypoint.bash b/k8s/nginx-https/container/nginx_entrypoint.bash new file mode 100755 index 0000000..9094202 --- /dev/null +++ b/k8s/nginx-https/container/nginx_entrypoint.bash @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +# Authorization Modes +threescale_auth_mode="threescale" +secret_token_auth_mode="secret-token" + + +# Cluster vars +node_fqdn=`printenv NODE_FQDN` +node_frontend_port=`printenv NODE_FRONTEND_PORT` + + +# NGINX vars +dns_server=`printenv DNS_SERVER` +health_check_port=`printenv HEALTH_CHECK_PORT` +authorization_mode=`printenv AUTHORIZATION_MODE` + +# MongoDB vars +mongo_backend_host=`printenv MONGODB_BACKEND_HOST` +mongo_backend_port=`printenv MONGODB_BACKEND_PORT` + +# OpenResty vars +openresty_backend_host=`printenv OPENRESTY_BACKEND_HOST` +openresty_backend_port=`printenv OPENRESTY_BACKEND_PORT` + +# Planetmint vars +bdb_backend_host=`printenv PLANETMINT_BACKEND_HOST` +bdb_api_port=`printenv PLANETMINT_API_PORT` +bdb_ws_port=`printenv PLANETMINT_WS_PORT` + +# Tendermint vars +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` +tm_p2p_port=`printenv TM_P2P_PORT` + + +# sanity check +if [[ -z "${node_frontend_port:?NODE_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_backend_host:?MONGODB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${mongo_backend_port:?MONGODB_BACKEND_PORT not specified. Exiting!}" || \ + -z "${openresty_backend_port:?OPENRESTY_BACKEND_PORT not specified. Exiting!}" || \ + -z "${openresty_backend_host:?OPENRESTY_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_backend_host:?PLANETMINT_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?PLANETMINT_API_PORT not specified. Exiting!}" || \ + -z "${bdb_ws_port:?PLANETMINT_WS_PORT not specified. Exiting!}" || \ + -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \ + -z "${node_fqdn:?NODE_FQDN not specified. Exiting!}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then + echo "Missing required environment variables. Exiting!" + exit 1 +else + echo NODE_FQDN="$node_fqdn" + echo NODE_FRONTEND_PORT="$node_frontend_port" + echo DNS_SERVER="$dns_server" + echo HEALTH_CHECK_PORT="$health_check_port" + echo MONGODB_BACKEND_HOST="$mongo_backend_host" + echo MONGODB_BACKEND_PORT="$mongo_backend_port" + echo OPENRESTY_BACKEND_HOST="$openresty_backend_host" + echo OPENRESTY_BACKEND_PORT="$openresty_backend_port" + echo PLANETMINT_BACKEND_HOST="$bdb_backend_host" + echo PLANETMINT_API_PORT="$bdb_api_port" + echo PLANETMINT_WS_PORT="$bdb_ws_port" + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" + echo TM_P2P_PORT="$tm_p2p_port" +fi + +if [[ ${authorization_mode} == ${secret_token_auth_mode} ]]; then + NGINX_CONF_FILE=/etc/nginx/nginx.conf + secret_access_token=`printenv SECRET_ACCESS_TOKEN` + sed -i "s|SECRET_ACCESS_TOKEN|${secret_access_token}|g" ${NGINX_CONF_FILE} +elif [[ ${authorization_mode} == ${threescale_auth_mode} ]]; then + NGINX_CONF_FILE=/etc/nginx/nginx-threescale.conf + sed -i "s|OPENRESTY_BACKEND_PORT|${openresty_backend_port}|g" ${NGINX_CONF_FILE} + sed -i "s|OPENRESTY_BACKEND_HOST|${openresty_backend_host}|g" ${NGINX_CONF_FILE} +else + echo "Unrecognised authorization mode: ${authorization_mode}. Exiting!" + exit 1 +fi + +# configure the nginx.conf file with env variables +sed -i "s|NODE_FQDN|${node_fqdn}|g" ${NGINX_CONF_FILE} +sed -i "s|NODE_FRONTEND_PORT|${node_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c ${NGINX_CONF_FILE} diff --git a/k8s/nginx-https/nginx-https-dep.yaml b/k8s/nginx-https/nginx-https-dep.yaml new file mode 100644 index 0000000..3cb94c0 --- /dev/null +++ b/k8s/nginx-https/nginx-https-dep.yaml @@ -0,0 +1,135 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: ngx-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: bigchaindb/nginx_https:2.2.2 + imagePullPolicy: Always + env: + - name: NODE_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: node-frontend-port + - name: HEALTH_CHECK_PORT + valueFrom: + configMapKeyRef: + name: vars + key: node-health-check-port + - name: NODE_FQDN + valueFrom: + configMapKeyRef: + name: vars + key: node-fqdn + - name: DNS_SERVER + valueFrom: + configMapKeyRef: + name: vars + key: node-dns-server-ip + - name: MONGODB_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-mdb-instance-name + - name: MONGODB_BACKEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: OPENRESTY_BACKEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: openresty-backend-port + - name: OPENRESTY_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-openresty-instance-name + - name: PLANETMINT_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-bdb-instance-name + - name: PLANETMINT_API_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-api-port + - name: PLANETMINT_WS_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-port + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-pub-key-access + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: bdb-p2p-port + - name: AUTHORIZATION_MODE + valueFrom: + configMapKeyRef: + name: vars + key: authorization-mode + - name: SECRET_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: nginx-secret-header + key: secret-token + ports: + # return a pretty error message on port 80, since we are expecting + # HTTPS traffic. + - containerPort: 80 + protocol: TCP + - containerPort: 443 + protocol: TCP + - containerPort: 8888 + protocol: TCP + name: ngx-port + - containerPort: 9986 + protocol: TCP + name: bdb-pub-key + - containerPort: 26656 + protocol: TCP + name: bdb-p2p-port + livenessProbe: + httpGet: + path: /health + port: ngx-port + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + volumeMounts: + - name: https-certs + mountPath: /etc/nginx/ssl/ + readOnly: true + restartPolicy: Always + volumes: + - name: https-certs + secret: + secretName: https-certs + defaultMode: 0400 diff --git a/k8s/nginx-https/nginx-https-svc.yaml b/k8s/nginx-https/nginx-https-svc.yaml new file mode 100644 index 0000000..1bcc08e --- /dev/null +++ b/k8s/nginx-https/nginx-https-svc.yaml @@ -0,0 +1,41 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: ngx-instance-0 + namespace: default + labels: + name: ngx-instance-0 + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-instance-0-dep + ports: + - port: 443 + targetPort: 443 + name: public-secure-node-port + protocol: TCP + - port: 27017 + targetPort: 27017 + name: public-mdb-port + protocol: TCP + - port: 9986 + targetPort: 9986 + name: tm-pub-key-access + protocol: TCP + - port: 26656 + targetPort: 26656 + protocol: TCP + name: tm-p2p-port + - port: 80 + targetPort: 80 + name: public-insecure-node-port + protocol: TCP + type: LoadBalancer diff --git a/k8s/nginx-openresty/LICENSE.md b/k8s/nginx-openresty/LICENSE.md new file mode 100644 index 0000000..d5e58e0 --- /dev/null +++ b/k8s/nginx-openresty/LICENSE.md @@ -0,0 +1,70 @@ +# Licenses on the Stuff in this Directory + +All _code_ in this directory is copyright Planetmint GmbH, +except for the configuration files obtained from 3scale (NGINX configuration +file and NGINX Lua script). + +`nginx.conf.template` and `nginx.lua.template` are based on files we got from +3scale (by going to the 3scale admin site - API - Integration - Download +the NGINX Config files). + +The original files (from 3scale) were licensed under an MIT License, +the text of which can be found below. + +The derived files (`nginx.conf.template` and `nginx.lua.template`), along with +the other files in this directory, are _also_ licensed under an MIT License, +the text of which can be found below. + +## Documentation Licenses + +The documentation in this directory is licensed under a Creative Commons Attribution +4.0 International license, the full text of which can be found at +[http://creativecommons.org/licenses/by/4.0/legalcode](http://creativecommons.org/licenses/by/4.0/legalcode). + +
+ +The MIT License + +Copyright (c) 2016-2017 Planetmint GmbH. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +
+ +The MIT License + +Copyright (c) 2007-2016 3scale Networks S.L. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/k8s/nginx-openresty/container/Dockerfile b/k8s/nginx-openresty/container/Dockerfile new file mode 100644 index 0000000..78a4485 --- /dev/null +++ b/k8s/nginx-openresty/container/Dockerfile @@ -0,0 +1,15 @@ +FROM openresty/openresty:xenial +LABEL maintainer "contact@ipdb.global" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /usr/local/openresty/nginx/conf/nginx.conf +COPY nginx.lua.template /usr/local/openresty/nginx/conf/nginx.lua +COPY nginx_openresty_entrypoint.bash / +# The following ports are the values we use to run the NGINX+3scale container. +# 80 for http, 8080 for the 3scale api, 8888 for health-check, 27017 for +# MongoDB +EXPOSE 80 8080 8888 27017 +ENTRYPOINT ["/nginx_openresty_entrypoint.bash"] diff --git a/k8s/nginx-openresty/container/README.md b/k8s/nginx-openresty/container/README.md new file mode 100644 index 0000000..d74e892 --- /dev/null +++ b/k8s/nginx-openresty/container/README.md @@ -0,0 +1,60 @@ +# nginx_3scale agent +nginx_3scale agent is a module that is responsible for providing authentication, +authorization and metering of Planetmint API users, by communicating with 3scale. +We use the openresty for this, which is nginx bundled with lua libraries. +More information at their [website](openresty.org/en) + +It validates the tokens sent by users in HTTP headers. +The user tokens map directly to the Application Plan specified in 3scale. + +## Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + + +## Working + +* We define a [lua module](./nginx.lua.template) and + custom hooks (lua functions to be executed at certain phases of the nginx + request processing lifecycle) to authenticate an API request. + +* Download the template available from 3scale which pre-defines all the + rules defined using the 3scale UI for monitoring, and the basic nginx + configuration. + +* We heavily modify these templates to add our custom functionality. + +* The nginx_3scale image reads the environment variables and accordingly + creates the nginx.conf and nginx.lua files from the templates. + +* Every request calls the `_M.access()` function. This function extracts the + `app_id` and `app_key` from the HTTP request headers and forwards it to + 3scale to see if a request is allowed to be forwarded to the Planetmint + backend. The request also contains the + various parameters that one would like to set access policies on. If the + `app_id` and `app_key` is successful, the access rules for the parameters + passed with the request are checked to see if the request can pass through. + For example, we can send a parameter, say `request_body_size`, to the 3scale + auth API. If we have defined a rule in the 3scale dashboard to drop + `request_body_size` above a certain threshold, the authorization will fail + even if the `app_id` and `app_key` are valid. + +* A successful response from the auth API causes the request to be proxied to + the backend. After a backend response, the `_M.post_action_content` hook is + called. We calculate details about all the metrics we are interested in and + form a payload for the 3scale reporting API. This ensures that we update + parameters of every metric defined in the 3scale UI after every request. + +* Note: We do not cache the keys in nginx so that we can validate every request + with 3scale and apply plan rules immediately. We can add auth caching to + improve performance, and in case we move to a fully post-paid billing model. + +* Refer to the references made in the [lua module](./nginx.lua.template) for + more details about how nginx+lua+3scale works + +* For HTTPS support, we also need to add the signed certificate and the + corresponding private key to the folder + `/usr/local/openresty/nginx/conf/ssl/`. Name the pem-encoded certificate as + `cert.pem` and the private key as `cert.key`. diff --git a/k8s/nginx-openresty/container/docker_build_and_push.bash b/k8s/nginx-openresty/container/docker_build_and_push.bash new file mode 100755 index 0000000..37033e5 --- /dev/null +++ b/k8s/nginx-openresty/container/docker_build_and_push.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +docker build -t bigchaindb/nginx_3scale:2.2.2 . + +docker push bigchaindb/nginx_3scale:2.2.2 diff --git a/k8s/nginx-openresty/container/nginx.conf.template b/k8s/nginx-openresty/container/nginx.conf.template new file mode 100644 index 0000000..92c0784 --- /dev/null +++ b/k8s/nginx-openresty/container/nginx.conf.template @@ -0,0 +1,197 @@ +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /usr/local/openresty/nginx/logs/error.log; +env THREESCALE_DEPLOYMENT_ENV; + +events { + worker_connections 256; + accept_mutex on; + use epoll; +} + +http { + lua_shared_dict api_keys 10m; + server_names_hash_bucket_size 128; + lua_package_path ";;$prefix/?.lua;$prefix/conf/?.lua"; + init_by_lua 'math.randomseed(ngx.time()) ; cjson = require("cjson")'; + access_log /usr/local/openresty/nginx/logs/access.log combined buffer=16k flush=5s; + + # allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + # enable logging when requests are being throttled + limit_req_log_level notice; + + # the http status code to return to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + resolver DNS_SERVER valid=30s ipv6=off; + + map $remote_addr $bdb_backend { + default PLANETMINT_BACKEND_HOST; + } + + upstream backend_SERVICE_ID { + server localhost:9999 max_fails=5 fail_timeout=30; + } + + # Our frontend API server that accepts requests from the external world and + # takes care of authentication and authorization. If auth is successful, it + # forwards the request to the backend_SERVICE_ID upstream where a consortium + # can run a BDB cluster. + server { + lua_code_cache on; + listen OPENRESTY_FRONTEND_PORT; + keepalive_timeout 60s; + + underscores_in_headers on; + set_by_lua $deployment 'return os.getenv("THREESCALE_DEPLOYMENT_ENV")'; + set $threescale_backend "https://su1.3scale.net"; + #set $threescale_backend "http://su1.3scale.net"; + #set $threescale_backend "https://su1.3scale.net:443"; + #set $threescale_backend "https://echo-api.3scale.net"; + + # `slowloris` attack mitigation settings + client_body_timeout 10s; + client_header_timeout 10s; + + location = /out_of_band_authrep_action { + internal; + proxy_pass_request_headers off; + set $service_token "SERVICE_TOKEN"; + content_by_lua "require('nginx').post_action_content()"; + } + + # 3scale auth api that takes the auth credentials and metrics as input, + # and returns 200 OK if both the credentials match and the user has not + # exceeded the limits in his application plan. + location = /threescale_auth { + internal; + set $service_token "SERVICE_TOKEN"; + proxy_pass $threescale_backend/transactions/authorize.xml?service_token=$service_token&service_id=$service_id&$usage&$credentials&log%5Bcode%5D=$arg_code&log%5Brequest%5D=$arg_req&log%5Bresponse%5D=$arg_resp; + proxy_set_header Host "su1.3scale.net"; + #proxy_set_header Host "echo-api.3scale.net"; + proxy_set_header X-3scale-User-Agent "nginx$deployment"; + proxy_set_header X-3scale-Version "THREESCALE_VERSION_HEADER"; + } + + # 3scale reporting api that takes the metrics data and persists the metrics + # in the 3scale backend. + location = /threescale_report { + internal; + set $service_token "SERVICE_TOKEN"; + proxy_pass $threescale_backend/transactions.xml; + proxy_set_header Host "su1.3scale.net"; + #proxy_set_header Host "echo-api.3scale.net"; + # We have a bug in lua-nginx module that does not set + # Content-Type from lua script + proxy_pass_request_headers off; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_set_header X-3scale-User-Agent "nginx$deployment"; + proxy_set_header X-3scale-Version "THREESCALE_VERSION_HEADER"; + } + + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-3scale-proxy-secret-token $secret_token; + + # limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # We do not need the GET handling here as it's done in the other NGINX + # module + #if ($request_method = GET ) { + # proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + #} + + if ($request_method = POST ) { + set $service_token null; + set $cached_key null; + set $credentials null; + set $usage null; + set $service_id SERVICE_ID; + set $proxy_pass null; + set $secret_token null; + set $resp_body null; + set $resp_headers null; + access_by_lua "require('nginx').access()"; + body_filter_by_lua 'ngx.ctx.buffered = (ngx.ctx.buffered or "") .. string.sub(ngx.arg[1], 1, 1000) + if ngx.arg[2] then ngx.var.resp_body = ngx.ctx.buffered end'; + header_filter_by_lua 'ngx.var.resp_headers = cjson.encode(ngx.resp.get_headers())'; + + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass $proxy_pass ; + post_action /out_of_band_authrep_action; + } + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + } + } + + # Our backend server block that accepts requests from the nginx proxy and + # forwards it to instances of BDB cluster. We currently run only a single + # instance. + server { + sendfile on; + + listen 9999; + + # max client request body size: avg transaction size + client_max_body_size 15k; + + # keepalive connection settings + keepalive_timeout 60s; + + # `slowloris` attack mitigation settings + client_body_timeout 10s; + client_header_timeout 10s; + + if ( $http_x_3scale_proxy_secret_token != "THREESCALE_RESPONSE_SECRET_TOKEN" ) { + return 403; + } + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # enable the following line if and only if you use HTTPS + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://$bdb_backend:PLANETMINT_API_PORT; + + # limit requests from the same client, allow `burst` to 20 r/s on avg, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/local/openresty/nginx/html/50x.html; + } + } +} diff --git a/k8s/nginx-openresty/container/nginx.lua.template b/k8s/nginx-openresty/container/nginx.lua.template new file mode 100644 index 0000000..64baeaa --- /dev/null +++ b/k8s/nginx-openresty/container/nginx.lua.template @@ -0,0 +1,416 @@ +-- -*- mode: lua; -*- +-- Generated on: 2017-04-10 14:41:18 +0000 -- +-- Version: +-- Error Messages per service + +-- Ref: https://github.com/openresty/lua-nginx-module +-- Ref: https://ipdbtestnet-admin.3scale.net/p/admin/api_docs +-- Ref: http://nginx.org/en/docs/debugging_log.html + +local custom_config = false + +local _M = { + ['services'] = { + ['SERVICE_ID'] = { + error_auth_failed = 'Authentication failed', + error_auth_missing = 'Authentication parameters missing', + auth_failed_headers = 'text/plain; charset=us-ascii', + auth_missing_headers = 'text/plain; charset=us-ascii', + error_no_match = 'No Mapping Rule matched', + no_match_headers = 'text/plain; charset=us-ascii', + no_match_status = 404, + auth_failed_status = 403, + auth_missing_status = 403, + secret_token = 'THREESCALE_RESPONSE_SECRET_TOKEN', + get_credentials = function(service, params) + return ( + (params.app_id and params.app_key) + ) or error_no_credentials(service) + end, + extract_usage = function (service, request) + local method, url = unpack(string.split(request," ")) + local path, querystring = unpack(string.split(url, "?")) + local usage_t = {} + local matched_rules = {} + + local args = get_auth_params(nil, method) + + for i,r in ipairs(service.rules) do + check_rule({path=path, method=method, args=args}, r, usage_t, matched_rules) + end + + -- if there was no match, usage is set to nil and it will respond a 404, this behavior can be changed + return usage_t, table.concat(matched_rules, ", ") + end, + rules = { + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'hits', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'request_body_size', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'response_body_size', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'post_transactions', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'total_body_size', + delta = 1 + }, + } +}, + } +} + +-- Error Codes +function error_no_credentials(service) + ngx.status = service.auth_missing_status + ngx.header.content_type = service.auth_missing_headers + ngx.print(service.error_auth_missing) + ngx.exit(ngx.HTTP_OK) +end + +function error_authorization_failed(service) + ngx.status = service.auth_failed_status + ngx.header.content_type = service.auth_failed_headers + ngx.print(service.error_auth_failed) + ngx.exit(ngx.HTTP_OK) +end + +function error_no_match(service) + ngx.status = service.no_match_status + ngx.header.content_type = service.no_match_headers + ngx.print(service.error_no_match) + ngx.exit(ngx.HTTP_OK) +end +-- End Error Codes + +-- Aux function to split a string + +function string:split(delimiter) + local result = { } + local from = 1 + local delim_from, delim_to = string.find( self, delimiter, from ) + if delim_from == nil then return {self} end + while delim_from do + table.insert( result, string.sub( self, from , delim_from-1 ) ) + from = delim_to + 1 + delim_from, delim_to = string.find( self, delimiter, from ) + end + table.insert( result, string.sub( self, from ) ) + return result +end + +function first_values(a) + r = {} + for k,v in pairs(a) do + if type(v) == "table" then + r[k] = v[1] + else + r[k] = v + end + end + return r +end + +function set_or_inc(t, name, delta) + return (t[name] or 0) + delta +end + +function build_querystring_formatter(fmt) + return function (query) + local function kvmap(f, t) + local res = {} + for k, v in pairs(t) do + table.insert(res, f(k, v)) + end + return res + end + + return table.concat(kvmap(function(k,v) return string.format(fmt, k, v) end, query or {}), "&") + end +end + +local build_querystring = build_querystring_formatter("usage[%s]=%s") +local build_query = build_querystring_formatter("%s=%s") + +function regexpify(path) + return path:gsub('?.*', ''):gsub("{.-}", '([\\w_.-]+)'):gsub("%.", "\\.") +end + +function check_rule(req, rule, usage_t, matched_rules) + local param = {} + local p = regexpify(rule.pattern) + local m = ngx.re.match(req.path, + string.format("^%s",p)) + if m and req.method == rule.method then + local args = req.args + if rule.querystring_params(args) then -- may return an empty table + -- when no querystringparams + -- in the rule. it's fine + for i,p in ipairs(rule.parameters) do + param[p] = m[i] + end + + table.insert(matched_rules, rule.pattern) + usage_t[rule.system_name] = set_or_inc(usage_t, rule.system_name, rule.delta) + end + end +end + +--[[ + Authorization logic + NOTE: We do not use any of the authorization logic defined in the template. + We use custom authentication and authorization logic defined in the + custom_app_id_authorize() function. +]]-- + +function get_auth_params(where, method) + local params = {} + if where == "headers" then + params = ngx.req.get_headers() + elseif method == "GET" then + params = ngx.req.get_uri_args() + else + ngx.req.read_body() + params = ngx.req.get_post_args() + end + return first_values(params) +end + +function get_debug_value() + local h = ngx.req.get_headers() + if h["X-3scale-debug"] == 'SERVICE_TOKEN' then + return true + else + return false + end +end + +function _M.authorize(auth_strat, params, service) + if auth_strat == 'oauth' then + oauth(params, service) + else + authrep(params, service) + end +end + +function oauth(params, service) + ngx.var.cached_key = ngx.var.cached_key .. ":" .. ngx.var.usage + local access_tokens = ngx.shared.api_keys + local is_known = access_tokens:get(ngx.var.cached_key) + + if is_known ~= 200 then + local res = ngx.location.capture("/threescale_oauth_authrep", { share_all_vars = true }) + + -- IN HERE YOU DEFINE THE ERROR IF CREDENTIALS ARE PASSED, BUT THEY ARE NOT VALID + if res.status ~= 200 then + access_tokens:delete(ngx.var.cached_key) + ngx.status = res.status + ngx.header.content_type = "application/json" + ngx.var.cached_key = nil + error_authorization_failed(service) + else + access_tokens:set(ngx.var.cached_key,200) + end + + ngx.var.cached_key = nil + end +end + +function authrep(params, service) + ngx.var.cached_key = ngx.var.cached_key .. ":" .. ngx.var.usage + local api_keys = ngx.shared.api_keys + local is_known = api_keys:get(ngx.var.cached_key) + + if is_known ~= 200 then + local res = ngx.location.capture("/threescale_authrep", { share_all_vars = true }) + + -- IN HERE YOU DEFINE THE ERROR IF CREDENTIALS ARE PASSED, BUT THEY ARE NOT VALID + if res.status ~= 200 then + -- remove the key, if it's not 200 let's go the slow route, to 3scale's backend + api_keys:delete(ngx.var.cached_key) + ngx.status = res.status + ngx.header.content_type = "application/json" + ngx.var.cached_key = nil + error_authorization_failed(service) + else + api_keys:set(ngx.var.cached_key,200) + end + ngx.var.cached_key = nil + end +end + +function _M.access() + local params = {} + local host = ngx.req.get_headers()["Host"] + local auth_strat = "" + local service = {} + local usage = {} + local matched_patterns = '' + + if ngx.status == 403 then + ngx.say("Throttling due to too many requests") + ngx.exit(403) + end + + if ngx.var.service_id == 'SERVICE_ID' then + local parameters = get_auth_params("headers", string.split(ngx.var.request, " ")[1] ) + service = _M.services['SERVICE_ID'] -- + ngx.var.secret_token = service.secret_token + params.app_id = parameters["app_id"] + params.app_key = parameters["app_key"] -- or "" -- Uncoment the first part if you want to allow not passing app_key + service.get_credentials(service, params) + ngx.var.cached_key = "SERVICE_ID" .. ":" .. params.app_id ..":".. params.app_key + auth_strat = "2" + ngx.var.service_id = "SERVICE_ID" + ngx.var.proxy_pass = "http://backend_SERVICE_ID" + usage, matched_patterns = service:extract_usage(ngx.var.request) + end + + usage['post_transactions'] = 0 + usage['request_body_size'] = 0 + usage['total_body_size'] = 0 + usage['response_body_size'] = 0 + ngx.var.credentials = build_query(params) + ngx.var.usage = build_querystring(usage) + + -- WHAT TO DO IF NO USAGE CAN BE DERIVED FROM THE REQUEST. + if ngx.var.usage == '' then + ngx.header["X-3scale-matched-rules"] = '' + error_no_match(service) + end + + if get_debug_value() then + ngx.header["X-3scale-matched-rules"] = matched_patterns + ngx.header["X-3scale-credentials"] = ngx.var.credentials + ngx.header["X-3scale-usage"] = ngx.var.usage + ngx.header["X-3scale-hostname"] = ngx.var.hostname + end + _M.custom_app_id_authorize(params, service) +end + +function _M.custom_app_id_authorize(params, service) + ngx.var.cached_key = ngx.var.cached_key .. ":" .. ngx.var.usage + local api_keys = ngx.shared.api_keys + local res = ngx.location.capture("/threescale_auth", { share_all_vars = true }) + if res.status ~= 200 then + ngx.status = res.status + ngx.header.content_type = "application/json" + ngx.var.cached_key = nil + error_authorization_failed(service) + end + ngx.var.cached_key = nil +end + +function _M.post_action_content() + local report_data = {} + + -- increment POST count + report_data['post_transactions'] = 1 + + -- NOTE: When we are querying for the length of the request here, we already + -- have the complete request data with us and hence can just use the len() + -- function to get the size of the payload in bytes. + -- However, we might not have a complete response from the backend at this + -- stage (esp. if it's a large response size). So, we decipher the payload + -- size by peeking into the content length header of the response. + -- Otherwise, nginx will have to buffer every response and then calculate + -- response payload size. + + -- req data size + local req_data = ngx.req.get_body_data() + if req_data then + report_data['request_body_size'] = req_data:len() + else + report_data['request_body_size'] = 0 + end + + -- res data size + local all_headers = cjson.decode(ngx.var.resp_headers) + local variable_header = "content-length" --<-- case sensitive + if all_headers[variable_header] then + report_data['response_body_size'] = all_headers[variable_header] + else + report_data['response_body_size'] = 0 + end + + -- total data size + report_data['total_body_size'] = report_data['request_body_size'] + report_data['response_body_size'] + + -- get the app_id + local app_id = "" + local credentials = ngx.var.credentials:split("&") + for i in pairs(credentials) do + if credentials[i]:match('app_id') then + local temp = credentials[i]:split("=") + app_id = temp[2] + end + end + + -- form the payload to report to 3scale + local report = {} + report['service_id'] = ngx.var.service_id + report['service_token'] = ngx.var.service_token + report['transactions[0][app_id]'] = app_id + report['transactions[0][usage][post_transactions]'] = report_data['post_transactions'] + report['transactions[0][usage][request_body_size]'] = report_data['request_body_size'] + report['transactions[0][usage][response_body_size]'] = report_data['response_body_size'] + report['transactions[0][usage][total_body_size]'] = report_data['total_body_size'] + local res1 = ngx.location.capture("/threescale_report", {method = ngx.HTTP_POST, body = ngx.encode_args(report), share_all_vars = true }) + --ngx.log(0, ngx.encode_args(report)) + ngx.log(0, "Status: "..res1.status) + ngx.log(0, "Body: "..res1.body) + --if res1.status ~= 200 then + -- local api_keys = ngx.shared.api_keys + -- api_keys:delete(cached_key) + --end + ngx.exit(ngx.HTTP_OK) +end + +if custom_config then + local ok, c = pcall(function() return require(custom_config) end) + if ok and type(c) == 'table' and type(c.setup) == 'function' then + c.setup(_M) + end +end + +return _M + +-- END OF SCRIPT diff --git a/k8s/nginx-openresty/container/nginx_openresty_entrypoint.bash b/k8s/nginx-openresty/container/nginx_openresty_entrypoint.bash new file mode 100755 index 0000000..c517691 --- /dev/null +++ b/k8s/nginx-openresty/container/nginx_openresty_entrypoint.bash @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +# Openresty vars +dns_server=`printenv DNS_SERVER` +openresty_frontend_port=`printenv OPENRESTY_FRONTEND_PORT` + + +# Planetmint vars +bdb_backend_host=`printenv PLANETMINT_BACKEND_HOST` +bdb_api_port=`printenv PLANETMINT_API_PORT` + + +# Read the 3scale credentials from the mountpoint +# Should be mounted at the following directory +THREESCALE_CREDENTIALS_DIR=/usr/local/openresty/nginx/conf/threescale + +threescale_secret_token=`cat ${THREESCALE_CREDENTIALS_DIR}/secret-token` +threescale_service_id=`cat ${THREESCALE_CREDENTIALS_DIR}/service-id` +threescale_version_header=`cat ${THREESCALE_CREDENTIALS_DIR}/version-header` +threescale_service_token=`cat ${THREESCALE_CREDENTIALS_DIR}/service-token` + + +if [[ -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${openresty_frontend_port:?OPENRESTY_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${bdb_backend_host:?PLANETMINT_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?PLANETMINT_API_PORT not specified. Exiting!}" || \ + -z "${threescale_secret_token:?3scale secret token not specified. Exiting!}" || \ + -z "${threescale_service_id:?3scale service id not specified. Exiting!}" || \ + -z "${threescale_version_header:?3scale version header not specified. Exiting!}" || \ + -z "${threescale_service_token:?3scale service token not specified. Exiting!}" ]]; then + echo "Invalid environment settings detected. Exiting!" + exit 1 +fi + +NGINX_LUA_FILE=/usr/local/openresty/nginx/conf/nginx.lua +NGINX_CONF_FILE=/usr/local/openresty/nginx/conf/nginx.conf + +# configure the nginx.lua file with env variables +sed -i "s|SERVICE_ID|${threescale_service_id}|g" ${NGINX_LUA_FILE} +sed -i "s|THREESCALE_RESPONSE_SECRET_TOKEN|${threescale_secret_token}|g" ${NGINX_LUA_FILE} +sed -i "s|SERVICE_TOKEN|${threescale_service_token}|g" ${NGINX_LUA_FILE} + +# configure the nginx.conf file with env variables +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|OPENRESTY_FRONTEND_PORT|${openresty_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|PLANETMINT_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|THREESCALE_RESPONSE_SECRET_TOKEN|${threescale_secret_token}|g" $NGINX_CONF_FILE +sed -i "s|SERVICE_ID|${threescale_service_id}|g" $NGINX_CONF_FILE +sed -i "s|THREESCALE_VERSION_HEADER|${threescale_version_header}|g" $NGINX_CONF_FILE +sed -i "s|SERVICE_TOKEN|${threescale_service_token}|g" $NGINX_CONF_FILE + + +# start nginx +echo "INFO: starting nginx..." +exec /usr/local/openresty/nginx/sbin/nginx -c ${NGINX_CONF_FILE} diff --git a/k8s/nginx-openresty/nginx-openresty-dep.yaml b/k8s/nginx-openresty/nginx-openresty-dep.yaml new file mode 100644 index 0000000..eaf590e --- /dev/null +++ b/k8s/nginx-openresty/nginx-openresty-dep.yaml @@ -0,0 +1,69 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: openresty-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: openresty-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx-openresty + image: bigchaindb/nginx_3scale:2.2.2 + imagePullPolicy: IfNotPresent + env: + - name: DNS_SERVER + valueFrom: + configMapKeyRef: + name: vars + key: node-dns-server-ip + - name: OPENRESTY_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: openresty-backend-port + - name: PLANETMINT_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-bdb-instance-name + - name: PLANETMINT_API_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-api-port + ports: + - containerPort: "" + protocol: TCP + name: openresty-port + volumeMounts: + - name: threescale-credentials + mountPath: /usr/local/openresty/nginx/conf/threescale + readOnly: true + livenessProbe: + httpGet: + path: / + port: openresty-port + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always + volumes: + - name: threescale-credentials + secret: + secretName: threescale-credentials + defaultMode: 0400 diff --git a/k8s/nginx-openresty/nginx-openresty-svc.yaml b/k8s/nginx-openresty/nginx-openresty-svc.yaml new file mode 100644 index 0000000..c19706b --- /dev/null +++ b/k8s/nginx-openresty/nginx-openresty-svc.yaml @@ -0,0 +1,27 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +apiVersion: v1 +kind: Service +metadata: + name: openresty-instance-0 + namespace: default + labels: + name: openresty-instance-0 + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + # Ref: https://kubernetes.io/docs/tutorials/services/source-ip/ + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: openresty-instance-0-dep + ports: + - port: "" + targetPort: "" + name: openresty-svc-port + protocol: TCP + type: ClusterIP + clusterIP: None diff --git a/k8s/scripts/functions b/k8s/scripts/functions new file mode 100755 index 0000000..83ae087 --- /dev/null +++ b/k8s/scripts/functions @@ -0,0 +1,407 @@ +#!/usr/bin/env bash +set -euo pipefail + +function configure_root_ca(){ + # $1:- Base directory for Root CA + echo "Generate Root CA" + echo 'set_var EASYRSA_DN "org"' >> $1/vars + echo 'set_var EASYRSA_KEY_SIZE 4096' >> $1/vars + + #TODO: Parametrize the below configurations + echo 'set_var EASYRSA_REQ_COUNTRY "DE"' >> $1/vars + echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> $1/vars + echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> $1/vars + echo 'set_var EASYRSA_REQ_ORG "Planetmint GmbH"' >> $1/vars + echo 'set_var EASYRSA_REQ_OU "ROOT-CA"' >> $1/vars + echo 'set_var EASYRSA_REQ_EMAIL "contact@ipdb.global"' >> $1//vars + + sed -i.bk '/^extendedKeyUsage/ s/$/,clientAuth/' $1/x509-types/server + echo "set_var EASYRSA_SSL_CONF \"$1/openssl-1.0.cnf\"" >> $1/vars + echo "set_var EASYRSA_PKI \"$1/pki\"" >> $1/vars + echo "set_var EASYRSA_EXT_DIR \"$1/x509-types\"" >> $1/vars + $1/easyrsa init-pki + $1/easyrsa build-ca + $1/easyrsa gen-crl +} + +function configure_member_cert_gen(){ + # $1:- Base directory for MongoDB Member Requests/Keys + echo "Generate MongoDB Member Requests/Certificate(s)" + echo 'set_var EASYRSA_DN "org"' >> $1/vars + echo 'set_var EASYRSA_KEY_SIZE 4096' >> $1/vars + + #TODO: Parametrize the below configurations + echo 'set_var EASYRSA_REQ_COUNTRY "DE"' >> $1/vars + echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> $1/vars + echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> $1/vars + echo 'set_var EASYRSA_REQ_ORG "Planetmint GmbH"' >> $1/vars + echo 'set_var EASYRSA_REQ_OU "MONGO-MEMBER"' >> $1/vars + echo 'set_var EASYRSA_REQ_EMAIL "contact@ipdb.global"' >> $1/vars + echo "set_var EASYRSA_SSL_CONF \"$1/openssl-1.0.cnf\"" >> $1/vars + echo "set_var EASYRSA_PKI \"$1/pki\"" >> $1/vars + $1/easyrsa init-pki + $1/easyrsa --req-cn="$MDB_CN"-"$INDEX" --subject-alt-name=DNS:localhost,DNS:"$MDB_CN"-"$INDEX" gen-req "$MDB_CN"-"$INDEX" nopass +} + +function configure_client_cert_gen(){ + # $1:- Base directory for MongoDB Client Requests/Keys + echo "Generate MongoDB Client Requests/Certificate(s)" + echo 'set_var EASYRSA_DN "org"' >> $1/vars + echo 'set_var EASYRSA_KEY_SIZE 4096' >> $1/vars + + #TODO: Parametrize the below configurations + echo 'set_var EASYRSA_REQ_COUNTRY "DE"' >> $1/vars + echo 'set_var EASYRSA_REQ_PROVINCE "Berlin"' >> $1/vars + echo 'set_var EASYRSA_REQ_CITY "Berlin"' >> $1/vars + echo 'set_var EASYRSA_REQ_ORG "Planetmint GmbH"' >> $1/vars + echo 'set_var EASYRSA_REQ_OU "MONGO-CLIENT"' >> $1/vars + echo 'set_var EASYRSA_REQ_EMAIL "contact@ipdb.global"' >> $1/vars + echo "set_var EASYRSA_SSL_CONF \"$1/openssl-1.0.cnf\"" >> $1/vars + echo "set_var EASYRSA_PKI \"$1/pki\"" >> $1/vars + $1/easyrsa init-pki + $1/easyrsa gen-req "$BDB_CN" nopass + $1/easyrsa gen-req "$MDB_MON_CN"-"$INDEX" nopass +} + +function import_requests(){ + # $1:- Base directory for Root CA + $1/easyrsa import-req $BASE_MEMBER_CERT_DIR/$BASE_EASY_RSA_PATH/pki/reqs/"$MDB_CN"-"$INDEX".req "$MDB_CN"-"$INDEX" + $1/easyrsa import-req $BASE_CLIENT_CERT_DIR/$BASE_EASY_RSA_PATH/pki/reqs/"$BDB_CN".req "$BDB_CN" + $1/easyrsa import-req $BASE_CLIENT_CERT_DIR/$BASE_EASY_RSA_PATH/pki/reqs/"$MDB_MON_CN"-"$INDEX".req "$MDB_MON_CN"-"$INDEX" +} + +function sign_requests(){ + # $1:- Base directory for Root CA + $1/easyrsa --subject-alt-name=DNS:localhost,DNS:"$MDB_CN"-"$INDEX" sign-req server "$MDB_CN"-"$INDEX" + $1/easyrsa sign-req client "$BDB_CN" + $1/easyrsa sign-req client "$MDB_MON_CN"-"$INDEX" +} + +function make_pem_files(){ + # $1:- Base directory for Root CA + # $2:- Base directory for kubernetes related config for secret.yaml + mkdir $2 + cat $1/pki/issued/"$MDB_CN"-"$INDEX".crt $BASE_MEMBER_CERT_DIR/$BASE_EASY_RSA_PATH/pki/private/"$MDB_CN"-"$INDEX".key > $2/"$MDB_CN"-"$INDEX".pem + cat $1/pki/issued/"$BDB_CN".crt $BASE_CLIENT_CERT_DIR/$BASE_EASY_RSA_PATH/pki/private/"$BDB_CN".key > $2/"$BDB_CN".pem + cat $1/pki/issued/"$MDB_MON_CN"-"$INDEX".crt $BASE_CLIENT_CERT_DIR/$BASE_EASY_RSA_PATH/pki/private/"$MDB_MON_CN"-"$INDEX".key > $2/"$MDB_MON_CN"-"$INDEX".pem +} + +function convert_b64(){ + # $1:- Base directory for kubernetes related config for secret.yaml + # $2:- Base directory for Root CA + # $3:- Base directory for client requests/keys + cat $1/"$MDB_CN"-"$INDEX".pem | base64 -w 0 > $1/"$MDB_CN"-"$INDEX".pem.b64 + cat $1/"$BDB_CN".pem | base64 -w 0 > $1/"$BDB_CN".pem.b64 + cat $1/"$MDB_MON_CN"-"$INDEX".pem | base64 -w 0 > $1/"$MDB_MON_CN"-"$INDEX".pem.b64 + + cat $3/pki/private/"$BDB_CN".key | base64 -w 0 > $1/"$BDB_CN".key.b64 + cat $2/pki/ca.crt | base64 -w 0 > $1/ca.crt.b64 + cat $2/pki/crl.pem | base64 -w 0 > $1/crl.pem.b64 +} + +function configure_common(){ + apt-get update -y + apt-get install openssl -y + wget https://github.com/OpenVPN/easy-rsa/archive/3.0.1.tar.gz -P $1 + tar xzvf $1/3.0.1.tar.gz -C $1/ + rm $1/3.0.1.tar.gz + cp $1/$BASE_EASY_RSA_PATH/vars.example $1/$BASE_EASY_RSA_PATH/vars +} + +function get_users(){ + mkdir $1 + + openssl x509 -in $BASE_CA_DIR/$BASE_EASY_RSA_PATH/pki/issued/"$MDB_CN"-"$INDEX".crt -inform PEM -subject \ + -nameopt RFC2253 | head -n 1 | sed -r 's/^subject= //' > $1/"$MDB_CN"-"$INDEX".user + openssl x509 -in $BASE_CA_DIR/$BASE_EASY_RSA_PATH/pki/issued/"$BDB_CN".crt -inform PEM -subject \ + -nameopt RFC2253 | head -n 1 | sed -r 's/^subject= //' > $1/"$BDB_CN".user + openssl x509 -in $BASE_CA_DIR/$BASE_EASY_RSA_PATH/pki/issued/"$MDB_MON_CN"-"$INDEX".crt -inform PEM -subject \ + -nameopt RFC2253 | head -n 1 | sed -r 's/^subject= //' > $1/"$MDB_MON_CN"-"$INDEX".user + +} + +function generate_secretes_no_threescale(){ + # $1:- Base DIR for MongoDB certs + # #2:- Secret Token + # $3:- HTTPS certificate key file + # $4:- HTTPS certificate chain + + + mdb_instance_pem=`cat $1/"$MDB_CN"-"$INDEX".pem.b64` + bdb_instance_pem=`cat $1/"$BDB_CN".pem.b64` + bdb_instance_key=`cat $1/"$BDB_CN".key.b64` + root_ca_pem=`cat $1/ca.crt.b64` + root_crl_pem=`cat $1/crl.pem.b64` + + secrete_token=`echo $2 | base64 -w 0` + if [ -f $3 ]; then + https_cert_key=`cat $3 | base64 -w 0` + else + https_cert_key="" + fi + if [ -f $4 ]; then + https_cert_chain_pem=`cat $4 | base64 -w 0` + else + https_cert_chain_pem="" + fi + + mdb_admin_password=`echo $5 | base64 -w 0` + + + cat > secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: mdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-instance.pem: "${mdb_instance_pem}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: bdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded Planetmint instance certificate + bdb-instance.pem: "${bdb_instance_pem}" + # Base64-encoded private key (.key) + bdb-instance.key: "${bdb_instance_key}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: nginx-secret-header + namespace: default +type: Opaque +data: + # Base64-encoded secret token to authorize POST requests + secret-token: "${secrete_token}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-certs + namespace: default +type: Opaque +data: + # Base64-encoded HTTPS private key + cert.key: "${https_cert_key}" + # Base64-encoded HTTPS certificate chain + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert if from DigiCert, download "Best format for nginx". + cert.pem: "${https_cert_chain_pem}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ca-auth + namespace: default +type: Opaque +data: + # CA used to issue members/client certificates + # Base64-encoded CA certificate (ca.crt) + ca.pem: "${root_ca_pem}" + crl.pem: "${root_crl_pem}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-config + namespace: default +type: Opaque +data: + # Password for for MongoDB adminuser + mdb-admin-password: "${mdb_admin_password}" +EOF +} + +function generate_config_map(){ + + mdb_instance_name="$MDB_CN-$INDEX" + ngx_instance_name="ngx-instance-$INDEX" + + bdb_user=`cat "${1}"/"$BDB_CN".user` + mdb_admin_username="${2}" + node_fqdn="${3}" + bdb_persistent_peers="${4}" + bdb_validators="${5}" + bdb_validators_power="${6}" + bdb_genesis_time="${7}" + bdb_chain_id="${8}" + bdb_instance_name="${9}" + dns_resolver_k8s="${10}" + auth_mode="${11}" + node_frontend_port="${12}" + + cat > config-map.yaml << EOF +apiVersion: v1 +kind: ConfigMap +metadata: + name: vars + namespace: default +data: + # node-fqdn is the DNS name registered for your HTTPS certificate. + node-fqdn: "${node_fqdn}" + + # node-frontend-port is the port number on which this node's services + # are available to external clients. + node-frontend-port: "${node_frontend_port}" + + # node-health-check-port is the port number on which an external load + # balancer can check the status/liveness of the external/public server. + # In our deployment, Kubernetes sends 'livenessProbes' to this port and + # interprets a successful response as a 'healthy' service. + node-health-check-port: "8888" + + # node-dns-server-ip is the IP of the DNS server. A Kubernetes deployment + # always has a DNS server (kube-dns). + node-dns-server-ip: "${dns_resolver_k8s}" + + # mdb-instance-name is the name of the MongoDB instance in this Kubernetes cluster. + mdb-instance-name: "${mdb_instance_name}" + + # ngx-instance-name is the name of the NGINX instance in this Kubernetes cluster. + ngx-instance-name: "${ngx_instance_name}" + + # bdb-instance-name is the name of the Planetmint instance in this Kubernetes cluster. + bdb-instance-name: "${bdb_instance_name}" + + # openresty-instance-name is the name of the OpenResty instance in this + # Kubernetes cluster. + openresty-instance-name: "openresty-instance-0" + + # ngx-mdb-instance-name is the FQDN of the MongoDB instance in this + # Kubernetes cluster. + ngx-mdb-instance-name: "${mdb_instance_name}.default.svc.cluster.local" + + # ngx-bdb-instance-name is the FQDN of the Planetmint instance in this + # Kubernetes cluster. + ngx-bdb-instance-name: "${bdb_instance_name}.default.svc.cluster.local" + + # ngx-openresty-instance-name is the FQDN of the OpenResty instance in this + # Kubernetes cluster. + ngx-openresty-instance-name: "openresty-instance-0.default.svc.cluster.local" + + # mongodb-backend-port is the port on which MongoDB is actually + # available/listening for requests. + mongodb-backend-port: "27017" + + # openresty-backend-port is the port number on which OpenResty is listening + # for requests. This is used by the NGINX instance to forward the requests to + # the right port, and by OpenResty instance to bind to the correct port to + # receive requests from NGINX instance. + openresty-backend-port: "8080" + + # Planetmint configuration parameters + # Refer https://docs.bigchaindb.com/projects/server/en/latest/server-reference/configuration.html + + # bigchaindb-api-port is the port number on which Planetmint is listening + # for HTTP requests. + bigchaindb-api-port: "9984" + + # bigchaindb-server-bind is the socket where Planetmint binds for API + # requests. + bigchaindb-server-bind: "0.0.0.0:9984" + + # bigchaindb-ws-port and bigchaindb-ws-interface form the socket where + # Planetmint binds for Websocket connections. + bigchaindb-ws-port: "9985" + bigchaindb-ws-interface: "0.0.0.0" + + # bigchaindb-database-name is the database collection used by Planetmint with + # the MongoDB backend. + bigchaindb-database-name: "bigchain" + + # bigchaindb-wsserver-advertised-scheme is the protocol used to access the + # WebSocket API in Planetmint; can be 'ws' or 'wss' (default). + bigchaindb-wsserver-advertised-scheme: "wss" + + # Optional: Optimize storage engine(wired tiger) + # cache size. e.g. (2048MB, 2GB, 1TB), otherwise + # it will use the default cache size; i.e. max((50% RAM - 1GB), 256MB) + storage-engine-cache-size: "" + + # POST API authorization mode [threescale | secret-token] + authorization-mode: "${auth_mode}" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: bdb-config + namespace: default +data: + # Planetmint instance authentication user name + bdb-user: "${bdb_user}" + + # bigchaindb-database-maxtries is the maximum number of times that Planetmint + # will try to establish a connection with the database backend. + # If it is set to 0, then it will try forever. + bigchaindb-database-maxtries: "3" + + # bigchaindb-database-connection-timeout is the maximum number of + # milliseconds that Planetmint will wait before closing the connection while + # connecting to the database backend. + bigchaindb-database-connection-timeout: "5000" + + # bigchaindb-log-level is the log level used to log to the console. + bigchaindb-log-level: "debug" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tendermint-config + namespace: default +data: + # bdb-persistent-peers is the list of all the peers in the network. + bdb-persistent-peers: "${bdb_persistent_peers}" + + # bdb-validators is the list of all validators in the network. + bdb-validators: "${bdb_validators}" + + # bdb-validator-power is the validators voting power, make sure the order and + # the number of nodes in bdb-validator-power and bdb-validators is the same. + bdb-validator-power: "${bdb_validators_power}" + + # bdb-genesis-time is the official time of blockchain start. + # example: 0001-01-01T00:00:00Z + bdb-genesis-time: "${bdb_genesis_time}" + + # bdb-chain-id is the ID of the blockchain. Must be unique for every blockchain. + # example: test-chain-KPI1Ud + bdb-chain-id: "${bdb_chain_id}" + + # bdb-abci-port is used by Tendermint Core for ABCI traffic. Planetmint nodes + # use that internally. + bdb-abci-port: "26658" + + # bdb-p2p-port is used by Tendermint Core to communicate with + # other peers in the network. This port is accessible publicly. + bdb-p2p-port: "26656" + + # bdb-rpc-port is used by Tendermint Core to rpc. Planetmint nodes + # use this port internally. + bdb-rpc-port: "26657" + + # bdb-pub-key-access is the port number used to host/publish the + # public key of the tendemrint node in this cluster. + bdb-pub-key-access: "9986" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mdb-config + namespace: default +data: + # User name for MongoDB adminuser + mdb-admin-username: "${mdb_admin_username}" + mdb-mon-user: "" +EOF +} diff --git a/k8s/scripts/generate_configs.sh b/k8s/scripts/generate_configs.sh new file mode 100755 index 0000000..ddd8486 --- /dev/null +++ b/k8s/scripts/generate_configs.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -euo pipefail + +source vars +source functions + +# Default directory for certificates +CERT_DIR="certificates" + +# base variables with default values +MDB_CN="mdb-instance" +BDB_CN="$BDB_INSTANCE_NAME" +MDB_MON_CN="mdb-mon-instance" +INDEX='0' +CONFIGURE_CA='true' +CONFIGURE_MEMBER='true' +CONFIGURE_CLIENT='true' +SECRET_TOKEN=${SECRET_TOKEN:="secret-token"} +NODE_FRONTEND_PORT=${NODE_FRONTEND_PORT:="443"} + +function show_help(){ +cat > /dev/stdout << END +${0} --cert-dir - Name of directory containing certificates:- default ${CERT_DIR} +--help - show help +EXAMPLES +- "Generate configs" + ./generate_configs.sh --cert-dir ${CERT_DIR} +END +} + + +while [[ $# -gt 0 ]]; do + arg="$1" + case $arg in + --cert-dir) + CERT_DIR="$2" + shift + ;; + --help) + show_help + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift +done + +# sanity checks +if [[ -z "${CERT_DIR}" ]] ; then + echo "Missing required argument CERT_DIR" + exit 1 +fi + +if [[ -z "${AUTH_MODE}" ]]; then + echo "Missing required argument AUTH_MODE" + exit 1 +fi + +if [[ "${AUTH_MODE}" != "secret-token" && \ + "${AUTH_MODE}" != "threescale" ]]; then + echo "Invalid AUTH_MODE configuration, only accepted values are: [secret-token, threescale]" + exit 1 +fi + +# Create BASE_DIR +BASE_DIR="$(pwd)/${CERT_DIR}" +mkdir -p "${BASE_DIR}" + +BASE_CA_DIR="${BASE_DIR}"/bdb-node-ca +BASE_MEMBER_CERT_DIR="${BASE_DIR}"/member-cert +BASE_CLIENT_CERT_DIR="${BASE_DIR}"/client-cert +BASE_EASY_RSA_PATH='easy-rsa-3.0.1/easyrsa3' +BASE_K8S_DIR="${BASE_DIR}"/k8s +BASE_USERS_DIR="$BASE_DIR"/users + +# Configure Root CA +mkdir $BASE_CA_DIR +configure_common $BASE_CA_DIR +configure_root_ca $BASE_CA_DIR/$BASE_EASY_RSA_PATH + + +# Configure Member Request/Key generation +mkdir $BASE_MEMBER_CERT_DIR +configure_common $BASE_MEMBER_CERT_DIR +configure_member_cert_gen $BASE_MEMBER_CERT_DIR/$BASE_EASY_RSA_PATH + +# Configure Client Request/Key generation +mkdir $BASE_CLIENT_CERT_DIR +configure_common $BASE_CLIENT_CERT_DIR +configure_client_cert_gen $BASE_CLIENT_CERT_DIR/$BASE_EASY_RSA_PATH + +import_requests $BASE_CA_DIR/$BASE_EASY_RSA_PATH +sign_requests $BASE_CA_DIR/$BASE_EASY_RSA_PATH +make_pem_files $BASE_CA_DIR/$BASE_EASY_RSA_PATH $BASE_K8S_DIR +convert_b64 $BASE_K8S_DIR $BASE_CA_DIR/$BASE_EASY_RSA_PATH $BASE_CLIENT_CERT_DIR/$BASE_EASY_RSA_PATH + +get_users $BASE_USERS_DIR $BASE_CA_DIR/$BASE_EASY_RSA_PATH +generate_secretes_no_threescale $BASE_K8S_DIR $SECRET_TOKEN $HTTPS_CERT_KEY_FILE_NAME $HTTPS_CERT_CHAIN_FILE_NAME $MDB_ADMIN_PASSWORD + +generate_config_map $BASE_USERS_DIR $MDB_ADMIN_USER $NODE_FQDN $BDB_PERSISTENT_PEERS $BDB_VALIDATORS $BDB_VALIDATOR_POWERS $BDB_GENESIS_TIME \ + $BDB_CHAIN_ID $BDB_INSTANCE_NAME $NODE_DNS_SERVER $AUTH_MODE $NODE_FRONTEND_PORT diff --git a/k8s/scripts/vars b/k8s/scripts/vars new file mode 100644 index 0000000..6afd100 --- /dev/null +++ b/k8s/scripts/vars @@ -0,0 +1,53 @@ +# DNS name of the bigchaindb node +NODE_FQDN="test.bigchaindb.com" + +# NODE_FRONTEND_PORT is the port number on which this node's services +# are available to external clients. Default is 443(https) +NODE_FRONTEND_PORT="443" + +# Authorization mode: [secret-token, threescale] +AUTH_MODE="secret-token" + +# Secret token used for authorization of +# POST requests to the bigchaindb node +# Only required when AUTH_MODE=secret-token +SECRET_TOKEN="test-secret" + +# Absolute path for the SSL certificate key +HTTPS_CERT_KEY_FILE_NAME="/path/to/https.key" + +# Absolute path for the SSL certificate chain +HTTPS_CERT_CHAIN_FILE_NAME="/path/to/https.crt" + +# MongoDB Admin user credentials +MDB_ADMIN_USER="adminUser" +MDB_ADMIN_PASSWORD="superstrongpassword" + +# Planetmint instance name. This name should be unique +BDB_INSTANCE_NAME="bdb-instance-0" + +# Comma separated list of initial peers in the +# network. +BDB_PERSISTENT_PEERS="bdb-instance-0,bdb-instance-1,bdb-instance-2,bdb-instance-3" + +# Comma separated list of validators in the +# network +BDB_VALIDATORS="bdb-instance-0,bdb-instance-1,bdb-instance-2,bdb-instance-3" + +# Comma separated list of voting +# power of all validators. Make sure +# order and number of powers corresponds +# to BDB_VALIDATORS +BDB_VALIDATOR_POWERS="10,10,10,10" + +# Offical time of blockchain start +BDB_GENESIS_TIME="0001-01-01T00:00:00Z" + +# Blockchain ID must be unique for +# every blockchain +BDB_CHAIN_ID="test-chain-rwcPML" + +# IP Address of the resolver(DNS server). +# i.e. IP of `kube-dns`, can be retrieved using: +# $ kubectl get services --namespace=kube-system -l k8s-app=kube-dns +NODE_DNS_SERVER="10.0.0.10" diff --git a/k8s/toolbox/Dockerfile b/k8s/toolbox/Dockerfile new file mode 100644 index 0000000..84aff13 --- /dev/null +++ b/k8s/toolbox/Dockerfile @@ -0,0 +1,16 @@ +# Toolbox container for debugging +# Run as: +# docker run -it --rm --entrypoint sh bigchaindb/toolbox +# kubectl run -it toolbox --image bigchaindb/toolbox --restart=Never --rm + +FROM alpine:3.5 +LABEL maintainer "contact@ipdb.global" +WORKDIR / +RUN apk add --no-cache --update curl bind-tools python3-dev g++ \ + libffi-dev make vim git nodejs openssl-dev \ + && pip3 install ipython \ + && git clone https://github.com/bigchaindb/bigchaindb-driver \ + && cd bigchaindb-driver \ + && pip3 install -e . \ + && npm install -g wsc +ENTRYPOINT ["/bin/sh"] diff --git a/k8s/toolbox/README.md b/k8s/toolbox/README.md new file mode 100644 index 0000000..a058970 --- /dev/null +++ b/k8s/toolbox/README.md @@ -0,0 +1,21 @@ + + +## Docker container with debugging tools + +* curl +* bind-utils - provides nslookup, dig +* python3 +* make + +## Build + +`docker build -t bigchaindb/toolbox .` + +## Push + +`docker push bigchaindb/toolbox` diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..e9eff30 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,140 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# NOTE: You must replace example.testnet2.com +# with your node's actual DNS name in TWO PLACES below. + +# Frontend API server +worker_processes 4; +user nobody nogroup; +error_log /var/log/nginx/error.log; + +events { + # Each worker handles up to 512 connections. Increase this for heavy + # workloads. + worker_connections 64; + accept_mutex on; + use epoll; +} + +http { + access_log /var/log/nginx/access.log combined buffer=16k flush=5s; + + # Allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code that is returned to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 10s; + client_header_timeout 10s; + + # Do not expose NGINX data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + + keepalive_timeout 60s; + + # Frontend server for the external clients; acts as HTTPS termination point. + server { + listen 443 ssl; + server_name "example.testnet2.com"; + + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/cert.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + underscores_in_headers on; + + # Forward websockets directly to backend BDB. + location /api/v1/streams/valid_transactions { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://localhost:9985; + proxy_read_timeout 600s; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Forward other URL paths as per business logic/use case to BDB + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # max client request body size: average transaction size. + client_max_body_size 15k; + + # GET requests are forwarded to BDB. + if ($request_method = GET) { + proxy_pass http://localhost:9984; + } + + # POST requests: Enable CORS then forward to BDB. + if ($request_method = POST) { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass http://localhost:9984; + } + + # OPTIONS requests: Enable CORS and return 204. + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + # Only return this response if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen 9000; + + location = /health { + return 200; + } + } + + # Frontend server for the external clients; returns a pretty error message + # when an HTTP request is sent instead of HTTPS. + server { + listen 80; + server_name "example.testnet2.com"; + + location / { + add_header Upgrade "TLS/1.2, HTTP/1.1" always; + default_type text/plain; + return 426 'Consider using the HTTPS protocol next time!'; + } + } +} diff --git a/pkg/configuration/bigchaindb-start.yml b/pkg/configuration/bigchaindb-start.yml new file mode 100644 index 0000000..49f27c4 --- /dev/null +++ b/pkg/configuration/bigchaindb-start.yml @@ -0,0 +1,31 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +- hosts: all + gather_facts: false + vars_files: + - vars/stack-config.yml + vars: + action: "{{ operation }}" + home_dir: "{{ home_path }}" + serial: 1 + pre_tasks: + - raw: "apt-get -qqy update && apt-get install -qqy python-minimal python-pip || dnf -y install python27 || which python" + - name: Gathering Facts + setup: #aka gather_facts + roles: + - { role: py36, when: stack_type|lower == "local" } + - { role: docker, when: stack_type|lower == "docker" or stack_type|lower == "cloud" } + - mongodb + +- hosts: all + vars_files: + - vars/stack-config.yml + vars: + action: "{{ operation }}" + home_dir: "{{ home_path }}" + roles: + - tendermint + - bigchaindb diff --git a/pkg/configuration/bigchaindb-stop.yml b/pkg/configuration/bigchaindb-stop.yml new file mode 100644 index 0000000..b304a88 --- /dev/null +++ b/pkg/configuration/bigchaindb-stop.yml @@ -0,0 +1,20 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +- hosts: all + vars_files: + - vars/stack-config.yml + vars: + action: "{{ operation }}" + home_dir: "{{ home_path }}" + pre_tasks: + - raw: "apt-get -qqy update && apt-get install -qqy python-minimal python-pip || dnf -y install python27 || which python" + - name: Gathering Facts + setup: #aka gather_facts + roles: + - tendermint + - bigchaindb + - mongodb + - { role: docker, when: stack_type|lower == "docker" or stack_type|lower == "cloud" } diff --git a/pkg/configuration/group_vars/all b/pkg/configuration/group_vars/all new file mode 100644 index 0000000..f530443 --- /dev/null +++ b/pkg/configuration/group_vars/all @@ -0,0 +1,5 @@ +--- +ansible_connection: ssh +ansible_ssh_port: 22 +ansible_become: yes +ansible_ssh_common_args: '-o StrictHostKeyChecking=no' \ No newline at end of file diff --git a/pkg/configuration/host_vars/bdb-node-01 b/pkg/configuration/host_vars/bdb-node-01 new file mode 100644 index 0000000..fef41e4 --- /dev/null +++ b/pkg/configuration/host_vars/bdb-node-01 @@ -0,0 +1,5 @@ +# Place holder file for users, running Ansible playbooks manually. Otherwise Vagrant +# populates this dynamically. + +# Only needed for logging into remote hosts and adding host specific variables e.g. +#ansible_ssh_private_key_file: "/path/to/private/key" diff --git a/pkg/configuration/hosts/all b/pkg/configuration/hosts/all new file mode 100644 index 0000000..f84cc37 --- /dev/null +++ b/pkg/configuration/hosts/all @@ -0,0 +1,8 @@ +# Place holder file for users, running Ansible playbooks manually. Otherwise Vagrant +# populates this dynamically. + +# For local host +# ansible_connection=local + +# For remote host(s) +# ansible_ssh_user= ansible_sudo_pass= diff --git a/pkg/configuration/roles/bigchaindb/defaults/main.yml b/pkg/configuration/roles/bigchaindb/defaults/main.yml new file mode 100644 index 0000000..9e2335d --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/defaults/main.yml @@ -0,0 +1,48 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +dependencies_deb: + - g++ + - libffi-dev + - build-essential + - libssl-dev + +dependencies_yum: + - gcc-c++ + - "@Development Tools" + - libffi-devel + - openssl-devel + +dependencies_dnf: + - gcc-c++ + - redhat-rpm-config + - "@Development Tools" + - libffi-devel + - openssl-devel + +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" + +directories: + - /data + +backend_db: localmongodb #[localmongodb] + +bigchaindb_server_bind: "0.0.0.0:9984" + +# Docker configuration +bigchaindb_image_name: "bigchaindb/bigchaindb" +bigchaindb_image_tag: "develop" +bigchaindb_docker_name: "bigchaindb" +mongodb_docker_name: "mongodb" +tendermint_docker_name: "tendermint" +bigchaindb_default_server_port: 9984 +bigchaindb_default_ws_port: 9985 +bigchaindb_tendermint_port: 26657 +tendermint_abci_port: 45558 +bigchaindb_docker_net: "bigchaindb_docker_net" diff --git a/pkg/configuration/roles/bigchaindb/tasks/centos.yml b/pkg/configuration/roles/bigchaindb/tasks/centos.yml new file mode 100644 index 0000000..36b477d --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/centos.yml @@ -0,0 +1,24 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Creating directories | yum + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + tags: [bigchaindb] + +- name: Install dependencies | yum + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ dependencies_yum }}" + tags: [bigchaindb] + +- name: Install pip + shell: "easy_install-3.5 pip" \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/common.yml b/pkg/configuration/roles/bigchaindb/tasks/common.yml new file mode 100644 index 0000000..302410d --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/common.yml @@ -0,0 +1,46 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install pymongo + pip: + name: pymongo + state: present + tags: [bigchaindb] + +- name: Install Planetmint + shell: "python3.6 -m pip install -e /opt/stack/bigchaindb/.[dev] --ignore-installed pyyaml" + register: install_bdb + failed_when: "'FAILED' in install_bdb.stderr or install_bdb.rc != 0" + tags: [bigchaindb] + +- name: MongoDB Process Check + shell: pgrep mongod | wc -l + register: mdb_pchk + tags: [bigchaindb] + +- name: Tendermint Process Check + shell: pgrep tendermint | wc -l + register: tm_pchk + tags: [bigchaindb] + +- name: Planetmint Process Check + shell: pgrep bigchaindb | wc -l + register: bdb_pchk + tags: [bigchaindb] + +- name: Start Planetmint + shell: nohup bigchaindb -l DEBUG start > /tmp/bigchaindb_log_$(date +%Y%m%d_%H%M%S) 2>&1 & + environment: + PLANETMINT_DATABASE_BACKEND: "localmongodb" + PLANETMINT_DATABASE_HOST: "127.0.0.1" + PLANETMINT_DATABASE_PORT: "27017" + PLANETMINT_SERVER_BIND: "0.0.0.0:9984" + PLANETMINT_WSSERVER_HOST: "0.0.0.0" + PLANETMINT_WSSERVER_PORT: "9985" + PLANETMINT_TENDERMINT_HOST: "127.0.0.1" + PLANETMINT_TENDERMINT_PORT: "26657" + when: mdb_pchk.stdout| int != 0 and bdb_pchk.stdout| int == 0 and tm_pchk.stdout| int != 0 + tags: [bigchaindb] diff --git a/pkg/configuration/roles/bigchaindb/tasks/debian.yml b/pkg/configuration/roles/bigchaindb/tasks/debian.yml new file mode 100644 index 0000000..870a3d4 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/debian.yml @@ -0,0 +1,13 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install dependencies | apt + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ dependencies_deb }}" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml new file mode 100644 index 0000000..c1381b1 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml @@ -0,0 +1,12 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install dependencies | dnf + dnf: + name: "{{ item }}" + state: present + with_items: "{{ dependencies_dnf }}" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/main.yml b/pkg/configuration/roles/bigchaindb/tasks/main.yml new file mode 100644 index 0000000..6756c4c --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/main.yml @@ -0,0 +1,26 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: (stack_type|lower == "docker" or stack_type|lower == "cloud") and info_result is failed + tags: [bigchaindb] + +- import_tasks: start.yml + when: action|lower == 'start' + tags: [bigchaindb] + +- import_tasks: stop.yml + when: action|lower == 'stop' + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/start.yml b/pkg/configuration/roles/bigchaindb/tasks/start.yml new file mode 100644 index 0000000..0e80934 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/start.yml @@ -0,0 +1,62 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Building Planetmint Docker + docker_image: + name: "{{ bigchaindb_image_name }}" + state: build + tag: "{{bigchaindb_image_tag }}" + dockerfile: Dockerfile-alpine + path: "{{ home_dir }}/bigchaindb" + nocache: yes + buildargs: + backend: localmongodb + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- name: Start Planetmint Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}:{{ bigchaindb_image_tag }}" + hostname: "{{ bigchaindb_docker_name }}{{ item }}" + detach: true + network_mode: bridge + networks: + - name: "{{ bigchaindb_docker_net }}" + env: + PLANETMINT_DATABASE_BACKEND: "localmongodb" + PLANETMINT_DATABASE_HOST: "{{ mongodb_docker_name }}{{ item }}" + PLANETMINT_DATABASE_PORT: "27017" + PLANETMINT_SERVER_BIND: "0.0.0.0:9984" + PLANETMINT_WSSERVER_HOST: "0.0.0.0" + PLANETMINT_TENDERMINT_HOST: "{{ tendermint_docker_name }}{{ item }}" + PLANETMINT_TENDERMINT_PORT: "{{ bigchaindb_tendermint_port | string }}" + published_ports: + - "{{ bigchaindb_default_server_port }}" + - "{{ bigchaindb_default_ws_port }}" + - "{{ tendermint_abci_port }}" + restart_policy: always + state: started + command: bigchaindb -l DEBUG start + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- import_tasks: debian.yml + when: stack_type == "local" and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [bigchaindb] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [bigchaindb] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (distribution_name == "fedora") + tags: [bigchaindb] + +- import_tasks: common.yml + when: stack_type|lower == "local" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/stop.yml b/pkg/configuration/roles/bigchaindb/tasks/stop.yml new file mode 100644 index 0000000..d2c9100 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/stop.yml @@ -0,0 +1,39 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Stopping Planetmint Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}:{{ bigchaindb_image_tag }}" + keep_volumes: no + state: absent + force_kill: yes + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- name: Get Running Planetmint Process(es) + shell: "ps aux | grep \"[b]igchaindb\" | awk '{print $2}'" + register: bdb_ps + ignore_errors: yes + when: stack_type|lower == "local" + tags: [bigchaindb] + +- name: Kill Planetmint Process(es) + shell: "kill -9 {{ item }}" + with_items: "{{ bdb_ps.stdout_lines }}" + when: stack_type|lower == "local" and bdb_ps.stdout_lines|length > 0 + tags: [bigchaindb] + +- name: Kill auxiliary Planetmint Process(es) + shell: "pkill {{ item }}" + with_items: + - logging_server + - ws + - gunicorn + ignore_errors: yes + when: stack_type|lower == "local" + tags: [bigchaindb] diff --git a/pkg/configuration/roles/docker/defaults/main.yml b/pkg/configuration/roles/docker/defaults/main.yml new file mode 100644 index 0000000..6520eb4 --- /dev/null +++ b/pkg/configuration/roles/docker/defaults/main.yml @@ -0,0 +1,31 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +uninstall_old_version: false #[true, false] +docker_edition: 'ce' #[ce, ee] Currently, onlt CE is supported +docker_pkg: "docker-{{ docker_edition }}" #[docker-ce, docker-ee] +docker_update_channel: "stable" #[stable, edge] + +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" +server_arch: "amd64" #[amd64, armhf, s390x] + +# Docker Repositories +docker_apt_repo: "deb [arch={{ server_arch }}] https://download.docker.com/linux/{{ distribution_name }} {{ distribution_codename }} {{ docker_update_channel }}" +apt_key_fingerprint: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88" +apt_key_url: "https://download.docker.com/linux/{{ distribution_name }}/gpg" +docker_yum_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ distribution_major }}/$basearch/{{ docker_update_channel }}" +docker_dnf_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ docker_pkg }}.repo" + +# Epel repo +epel_repo_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" +epel_repo_gpg_key_url: "/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" +epel_repofile_path: "/etc/yum.repos.d/epel.repo" + +# Docker configuration +bigchaindb_docker_net: "bigchaindb_docker_net" diff --git a/pkg/configuration/roles/docker/tasks/centos.yml b/pkg/configuration/roles/docker/tasks/centos.yml new file mode 100644 index 0000000..2b6fbd5 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/centos.yml @@ -0,0 +1,71 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Uninstall older versions of Docker | CentOS + yum: + name: "{{ item }}" + state: absent + with_items: + - docker + - docker-common + - docker-engine + - docker-selinux + when: uninstall_old_version + tags: [docker] + +- name: Check if EPEL repo is already configured. + stat: path={{ epel_repofile_path }} + register: epel_repofile_result + tags: [docker] + +- name: Install EPEL repo. + yum: + name: "{{ epel_repo_url }}" + state: present + register: result + until: 'result.rc == 0' + retries: 5 + delay: 10 + when: not epel_repofile_result.stat.exists + tags: [docker] + +- name: Import EPEL GPG key. + rpm_key: + key: "{{ epel_repo_gpg_key_url }}" + state: present + when: not epel_repofile_result.stat.exists + ignore_errors: "{{ ansible_check_mode }}" + tags: [docker] + +- name: Setup Pre-reqs | CentOS + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - yum-utils + - device-mapper-persistent-data + - lvm2 + - python-pip + tags: [docker] + +- name: Add Docker Repo | CentOS + yum_repository: + name: "{{ docker_pkg }}" + gpgcheck: yes + gpgkey: https://download.docker.com/linux/centos/gpg + baseurl: "{{ docker_yum_repo }}" + file: "{{ docker_pkg }}" + description: "Docker Repo" + enabled: yes + tags: [docker] + +- name: Install Docker | CentOS + yum: + name: "{{ docker_pkg }}" + state: present + update_cache: yes + tags: [docker] diff --git a/pkg/configuration/roles/docker/tasks/debian.yml b/pkg/configuration/roles/docker/tasks/debian.yml new file mode 100644 index 0000000..6b29867 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/debian.yml @@ -0,0 +1,57 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Uninstall older versions of Docker | Debian + apt: + name: "{{ item }}" + state: absent + with_items: + - docker + - docker-engine + - docker.io + when: uninstall_old_version + tags: [docker] + +- name: Install dependencies | Debian + apt: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + - curl + - software-properties-common + - python3-pip + tags: [docker] + +- name: Add APT Key | Debian + apt_key: + url: "{{ apt_key_url }}" + id: "{{ apt_key_fingerprint }}" + state: present + register: add_repository_key + ignore_errors: true + tags: [docker] + +- name: Use curl if apt_key fails | Debian + shell: "curl -sSl {{ apt_key_url }} | sudo apt-key add -" + args: + warn: no + when: add_repository_key is failed + tags: [docker] + +- name: Add Docker repo and update cache | Debian + apt_repository: + repo: "{{ docker_apt_repo }}" + update_cache: yes + state: present + tags: [docker] + +- name: Install Docker | Debian + apt: + name: "{{ docker_pkg }}" + state: present + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/fedora.yml b/pkg/configuration/roles/docker/tasks/fedora.yml new file mode 100644 index 0000000..d880b93 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/fedora.yml @@ -0,0 +1,39 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Uninstall older versions of Docker | Fedora + dnf: + name: "{{ item }}" + state: absent + with_items: + - docker-engine-selinux + - docker-common + - docker-engine + - docker-selinux + when: uninstall_old_version + tags: [docker] + +- name: Setup Pre-reqs | Fedora + dnf: + name: "{{ item }}" + state: present + with_items: + - dnf-plugins-core + tags: [docker] + +- name: Add Docker repo | Fedora + shell: "dnf config-manager --add-repo {{ docker_dnf_repo }}" + tags: [docker] + +- name: Update Cache | Fedora + shell: "dnf makecache fast" + tags: [docker] + +- name: Install Docker | Fedora + dnf: + name: "{{ docker_pkg }}" + state: present + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/macos.yml b/pkg/configuration/roles/docker/tasks/macos.yml new file mode 100644 index 0000000..3651ef2 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/macos.yml @@ -0,0 +1,29 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check if Docker is Present + command: docker + register: result + ignore_errors: True + tags: [docker] + +- name: Error! Docker not found + fail: + msg: "Docker not installed on MacOSX, please visit: https://docs.docker.com/docker-for-mac/install/" + when: result is failed + tags: [docker] + +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + tags: [docker] + +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: info_result is failed + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/main.yml b/pkg/configuration/roles/docker/tasks/main.yml new file mode 100644 index 0000000..4fa455f --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/main.yml @@ -0,0 +1,79 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- import_tasks: debian.yml + when: distribution_name == "debian" or distribution_name == "ubuntu" + +- import_tasks: centos.yml + when: distribution_name == "centos" or distribution_name == "red hat enterprise linux" + +- import_tasks: fedora.yml + when: distribution_name == "fedora" + +- import_tasks: macos.yml + when: distribution_name == 'macosx' + +- name: Create Docker group + group: + name: docker + state: present + register: group_result + tags: [docker] + +- name: Add USER to docker group + user: + append: yes + name: "{{ item }}" + state: present + group: docker + with_items: + - "{{ ansible_hostname }}" + - "{{ distribution_name }}" + - "vagrant" + tags: [docker] + +- name: Start docker service | systemd + systemd: + name: docker + enabled: yes + state: started + when: (distribution_name != 'macosx') and distribution_name != 'debian' + tags: [docker] + +- name: Start docker service | init + service: + name: docker + state: started + enabled: yes + when: distribution_name == 'debian' + tags: [docker] + +- name: Install docker-py + pip: + name: docker-py + state: latest + tags: [docker] + +- name: Check if Planetmint Docker network exists + shell: docker inspect "{{ bigchaindb_docker_net }}" + register: network_ls + ignore_errors: True + tags: [docker] + + +- name: Create Planetmint Docker network + shell: docker network create "{{ bigchaindb_docker_net }}" --attachable + register: docker_net_create + when: (stack_type|lower == "docker" or stack_type == "cloud") and network_ls.rc != 0 and action|lower == 'start' + failed_when: "'FAILED' in docker_net_create.stderr or docker_net_create.rc != 0" + tags: [docker] + +- name: Delete Planetmint Docker network + shell: docker network rm "{{ bigchaindb_docker_net }}" + register: docker_net_delete + failed_when: "'FAILED' in docker_net_delete.stderr or docker_net_delete.rc != 0" + when: (stack_type|lower == "docker" or stack_type == "cloud") and network_ls.rc != 0 and action|lower == 'stop' + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/defaults/main.yml b/pkg/configuration/roles/mongodb/defaults/main.yml new file mode 100644 index 0000000..b89d080 --- /dev/null +++ b/pkg/configuration/roles/mongodb/defaults/main.yml @@ -0,0 +1,40 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +mongodb_package: "mongodb-org" +apt_key_fingerprint: "2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5" +apt_keyserver: "hkp://keyserver.ubuntu.com:80" +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" +server_arch: "amd64,arm64" + +# MongoDB Repos +mongodb_apt_repo: "deb [arch={{ server_arch }}] http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongo_version }} multiverse" +mongodb_deb_repo: "deb http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongo_version }} main" +mongodb_yum_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/$releasever/{{ mongodb_package }}/{{ mongo_version }}/{{ ansible_architecture }}" +mongodb_dnf_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/7/{{ mongodb_package }}/{{ mongo_version }}/{{ ansible_architecture }}" + +# Epel repo +epel_repo_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" +epel_repo_gpg_key_url: "/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" +epel_repofile_path: "/etc/yum.repos.d/epel.repo" + +# MongoDB running config +mongodb_storage_path: /data/db +mongodb_log_path: /var/log/mongodb +mongodb_config_path: /data/configdb +directories: + - "{{ mongodb_storage_path }}" + - "{{ mongodb_log_path }}" + - "{{ mongodb_config_path }}" + +mongodb_port: 27017 + +# Docker configuration +bigchaindb_docker_net: "bigchaindb_docker_net" +mongodb_docker_name: "mongodb" +mongodb_host_mount_dir: "{{ home_dir }}/mongodb_docker" diff --git a/pkg/configuration/roles/mongodb/tasks/centos.yml b/pkg/configuration/roles/mongodb/tasks/centos.yml new file mode 100644 index 0000000..e1b809a --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/centos.yml @@ -0,0 +1,55 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check if EPEL repo is already configured. + stat: path={{ epel_repofile_path }} + register: epel_repofile_result + tags: [mongodb] + +- name: Install EPEL repo. + yum: + name: "{{ epel_repo_url }}" + state: present + register: result + until: 'result.rc == 0' + retries: 5 + delay: 10 + when: not epel_repofile_result.stat.exists + tags: [mongodb] + +- name: Import EPEL GPG key. + rpm_key: + key: "{{ epel_repo_gpg_key_url }}" + state: present + when: not epel_repofile_result.stat.exists + ignore_errors: "{{ ansible_check_mode }}" + tags: [mongodb] + +- name: Add MongoDB Repo | yum + yum_repository: + name: "{{ mongodb_package }}-{{ mongo_version }}" + gpgcheck: yes + gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongo_version }}.asc + baseurl: "{{ mongodb_yum_base_url }}" + file: "{{ mongodb_package }}" + description: "MongoDB Repo" + enabled: yes + tags: [mongodb] + +- name: Install MongoDB | yum + yum: + name: "{{ mongodb_package }}" + state: present + update_cache: yes + tags: [mongodb] + +- name: Install pip | yum + yum: + name: python-pip + state: present + update_cache: yes + tags: [mongodb] + diff --git a/pkg/configuration/roles/mongodb/tasks/debian.yml b/pkg/configuration/roles/mongodb/tasks/debian.yml new file mode 100644 index 0000000..28274b2 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/debian.yml @@ -0,0 +1,48 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Removing apt cache lists + file: + path: /var/lib/apt/lists/* + state: absent + force: yes + tags: [mongodb] + +- name: Update Cache + apt: update_cache=yes + tags: [mongodb] + +- name: Add APT Key | apt + apt_key: + keyserver: "{{ apt_keyserver }}" + id: "{{ apt_key_fingerprint }}" + state: present + tags: [mongodb] + +- name: Add MongoDB repo and update cache | apt + apt_repository: + repo: "{{ mongodb_apt_repo }}" + state: present + update_cache: no + when: distribution_name == "ubuntu" + tags: [mongodb] + +- name: Add MongoDB repo and update cache | deb + apt_repository: + repo: "{{ mongodb_deb_repo }}" + state: present + update_cache: no + when: distribution_name == "debian" + tags: [mongodb] + +- name: Install MongoDB | apt + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ mongodb_package }}" + tags: [mongodb] diff --git a/pkg/configuration/roles/mongodb/tasks/fedora.yml b/pkg/configuration/roles/mongodb/tasks/fedora.yml new file mode 100644 index 0000000..26d0940 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/fedora.yml @@ -0,0 +1,24 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Add MongoDB Repo | dnf + yum_repository: + name: "{{ mongodb_package }}" + gpgcheck: yes + gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongo_version }}.asc + baseurl: "{{ mongodb_dnf_base_url }}" + file: "{{ mongodb_package }}" + description: "MongoDB Repo" + enabled: yes + tags: [mongodb] + +- name: Install MongoDB | dnf + dnf: + name: "{{ item }}" + state: present + with_items: + - "{{ mongodb_package }}" + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/main.yml b/pkg/configuration/roles/mongodb/tasks/main.yml new file mode 100644 index 0000000..2dc65a4 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/main.yml @@ -0,0 +1,24 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: (stack_type|lower == "docker" or stack_type|lower == "cloud") and info_result is failed + tags: [mongodb] + +- import_tasks: start.yml + when: action|lower == 'start' + +- import_tasks: stop.yml + when: action|lower == 'stop' diff --git a/pkg/configuration/roles/mongodb/tasks/start.yml b/pkg/configuration/roles/mongodb/tasks/start.yml new file mode 100644 index 0000000..28d0a78 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/start.yml @@ -0,0 +1,61 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Running MongoDB Docker + docker_container: + name: "{{ mongodb_docker_name }}{{ item }}" + hostname: "{{ mongodb_docker_name }}{{ item }}" + image: "mongo:{{ mongo_version }}" + detach: true + network_mode: bridge + networks: + - name: "{{ bigchaindb_docker_net }}" + published_ports: + - "{{ mongodb_port }}" + restart_policy: unless-stopped + volumes: + - "{{ mongodb_host_mount_dir }}{{ item|string }}/db:{{ mongodb_storage_path }}" + - "{{ mongodb_host_mount_dir }}{{ item|string }}/configdb:{{ mongodb_config_path }}" + state: started + keep_volumes: true + command: mongod + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Creating directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [mongodb] + +- import_tasks: debian.yml + when: stack_type|lower == "local" and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [mongodb] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [mongodb] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (distribution_name == "fedora") + tags: [mongodb] + +- name: MongoDB Process Check + shell: pgrep mongod | wc -l + register: command_result + when: stack_type|lower == "local" + tags: [mongodb] + +- name: Run MongoDB + shell: nohup mongod > /tmp/mongodb_log_$(date +%Y%m%d_%H%M%S) 2>&1 & + register: start_mdb + failed_when: "'FAILED' in start_mdb.stderr or start_mdb.rc != 0" + when: stack_type|lower == "local" and command_result.stdout| int == 0 + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/stop.yml b/pkg/configuration/roles/mongodb/tasks/stop.yml new file mode 100644 index 0000000..08c9025 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/stop.yml @@ -0,0 +1,46 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Stopping MongoDB Docker + docker_container: + name: "{{ mongodb_docker_name }}{{ item }}" + image: "mongo:{{ mongo_version }}" + state: absent + keep_volumes: no + force_kill: yes + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Removing MongoDB Directories + file: + path: "{{ mongodb_host_mount_dir }}{{ item }}" + state: absent + force: yes + with_sequence: start=0 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Get Running MongoDB Process + shell: "ps aux | grep \"[m]ongo\" | awk 'NR == 1 {print $2}'" + register: mongo_ps + when: stack_type|lower == "local" + tags: [mongodb] + +- name: Kill MongoDB Process + shell: "kill {{ item }}" + with_items: "{{ mongo_ps.stdout_lines }}" + when: stack_type|lower == "local" and mongo_ps.stdout_lines|length > 0 + tags: [mongodb] + +- name: Removing MongoDB Directories + file: + path: "{{ item }}" + state: absent + force: yes + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/centos.yml b/pkg/configuration/roles/py36/tasks/centos.yml new file mode 100644 index 0000000..2e10ea9 --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/centos.yml @@ -0,0 +1,26 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install dependencies py36 | yum + yum: + name: "{{ item }}" + update_cache: yes + state: present + with_items: + - "yum-utils" + - "https://centos7.iuscommunity.org/ius-release.rpm" + tags: [py36] + +- name: Install py36 | yum + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "python36u" + - "python36u-pip" + - "python36u-devel" + tags: [py36] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/debian.yml b/pkg/configuration/roles/py36/tasks/debian.yml new file mode 100644 index 0000000..6d4afdb --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/debian.yml @@ -0,0 +1,31 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check if python3 already installed + shell: which python3 + register: chk_py36 + ignore_errors: yes + tags: [py36] + +- name: Check version of python3 + shell: "python3 -c 'import platform; print(platform.python_version())' | cut -d. -f-2" + when: chk_py36.rc == 0 + register: py36_ver + tags: [py36] + +- name: Creating files for python 3.6 installation + file: + template: src=install_py36.j2 dest=/home/vagrant/install_py36.bash + mode: 0755 + when: chk_py36.rc == 0 and (py36_ver.stdout | float < 3.6) + tags: [py36] + +- name: Install py36 + shell: "bash /home/vagrant/install_py36.bash > install_py36.txt" + register: install_py36 + failed_when: "'FAILED' in install_py36.stderr or install_py36.rc != 0" + when: chk_py36.rc == 0 and (py36_ver.stdout | float < 3.6) + tags: [py36] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/fedora.yml b/pkg/configuration/roles/py36/tasks/fedora.yml new file mode 100644 index 0000000..8cac5cb --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/fedora.yml @@ -0,0 +1,44 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check version of python3 + shell: "python3 -c 'import platform; print(platform.python_version())' | cut -d. -f-2" + register: py36_ver + tags: [py36] + +- name: Install py36 | dnf + dnf: + name: "{{ item }}" + state: present + with_items: + - "python36" + - libselinux-python + - policycoreutils-python + - wget + when: py36_ver.stdout | float < 3.6 + tags: [py36] + +- name: Update fedora + shell: dnf -y update + register: update_fed + failed_when: "'FAILED' in update_fed.stderr or update_fed.rc != 0" + args: + warn: no + tags: [py36] + +- name: Creating files for python 3.6 installation + file: + template: src=install_pip36.j2 dest=/home/vagrant/install_pip36.bash + mode: 0755 + when: py36_ver.stdout | float < 3.6 + tags: [py36] + +- name: Install pip36 + shell: "bash /home/vagrant/install_pip36.bash > install_pip36.txt" + register: install_pip36 + failed_when: "'FAILED' in install_pip36.stderr or install_pip36.rc != 0" + when: py36_ver.stdout | float < 3.6 + tags: [py36] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/main.yml b/pkg/configuration/roles/py36/tasks/main.yml new file mode 100644 index 0000000..7461fe9 --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/main.yml @@ -0,0 +1,16 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- import_tasks: debian.yml + when: stack_type|lower == "local" and (ansible_distribution|lower == "debian" or ansible_distribution|lower == "ubuntu") + tags: [py36] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (ansible_distribution|lower == "centos" or ansible_distribution|lower == "red hat enterprise linux") + tags: [py36] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (ansible_distribution|lower == "fedora") \ No newline at end of file diff --git a/pkg/configuration/roles/py36/templates/install_pip36.j2 b/pkg/configuration/roles/py36/templates/install_pip36.j2 new file mode 100644 index 0000000..591e383 --- /dev/null +++ b/pkg/configuration/roles/py36/templates/install_pip36.j2 @@ -0,0 +1,8 @@ +{% raw %} +#!/bin/bash +set -ex +export PYTHON_PIP_VERSION=10.0.1 +wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py' +python3.6 get-pip.py --disable-pip-version-check +rm -f get-pip.py +{% endraw %} \ No newline at end of file diff --git a/pkg/configuration/roles/py36/templates/install_py36.j2 b/pkg/configuration/roles/py36/templates/install_py36.j2 new file mode 100644 index 0000000..4b5838c --- /dev/null +++ b/pkg/configuration/roles/py36/templates/install_py36.j2 @@ -0,0 +1,34 @@ +{% raw %} +#!/bin/bash +set -ex +export PATH=/usr/local/bin:$PATH +export LANG=C.UTF-8 +export GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D +export PYTHON_VERSION=3.6.5 +rm -rf /var/lib/apt/lists/* && \ +apt-get update && apt-get install -y --no-install-recommends tcl tk libssl-dev && rm -rf /var/lib/apt/lists/* && \ +apt-get update && apt-get install -y dpkg-dev tcl-dev tk-dev --no-install-recommends && rm -rf /var/lib/apt/lists/* && \ +wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" && \ +wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" && \ +export GNUPGHOME="$(mktemp -d)" && \ +gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" && \ +gpg --batch --verify python.tar.xz.asc python.tar.xz && \ +rm -rf "$GNUPGHOME" python.tar.xz.asc && \ +mkdir -p /usr/src/python && \ +tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz && \ +rm python.tar.xz && \ +cd /usr/src/python/ && \ +gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" && \ +./configure --build="$gnuArch" --enable-loadable-sqlite-extensions --enable-shared --with-system-expat --with-system-ffi && \ +make -j "$(nproc)" && \ +make install && \ +apt-get purge -y --auto-remove dpkg-dev tcl-dev tk-dev && \ +find /usr/local -depth \( \( -type d -a \( -name test -o -name tests \) \) -o \( -type f -a \( -name "*.pyc" -o -name "*.pyo" \) \) \) -exec rm -rf '{}' + && \ +cd $HOME && \ +rm -rf /usr/src/python +export PYTHON_PIP_VERSION=10.0.1 +wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py' +python3 get-pip.py --disable-pip-version-check "pip==$PYTHON_PIP_VERSION" +find /usr/local -depth \( \( -type d -a \( -name test -o -name tests \) \) -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \) -exec rm -rf '{}' +; +rm -f get-pip.py +{% endraw %} \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/defaults/main.yml b/pkg/configuration/roles/tendermint/defaults/main.yml new file mode 100644 index 0000000..6b621c1 --- /dev/null +++ b/pkg/configuration/roles/tendermint/defaults/main.yml @@ -0,0 +1,33 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +dependencies_key_exchange: + - nginx + - jq + +tendermint_binary_url: "https://github.com/tendermint/tendermint/releases/download/v{{ tm_version }}/tendermint_{{ tm_version }}_linux_amd64.zip" + +# Docker configuration +tendermint_image_name: "bigchaindb/tendermint" +tendermint_image_tag: "develop" +tendermint_docker_name: "tendermint" +bigchaindb_docker_name: "bigchaindb" +bigchaindb_docker_net: "bigchaindb_docker_net" +tendermint_host_mount_dir: "{{ home_dir }}/tendermint_docker" +tendermint_host_mount_config_dir: "{{ home_dir }}/tendermint_config" +tendermint_home: /tendermint/config +tendermint_data: /tendermint/data +tendermint_p2p_port: 26656 +tendermint_rpc_port: 26657 +tendermint_abci_port: 26658 + +directories: + - "{{ tendermint_home }}" + +tendermint_conf_files: [ + { src: "genesis.json", dest: "{{ tendermint_home }}/genesis.json"}, + { src: "config.toml", dest: "{{ tendermint_home}}/config.toml"}, + { src: "access_pub_key.conf", dest: "/etc/nginx/conf.d/access_pub_key.conf"} +] diff --git a/pkg/configuration/roles/tendermint/files/Dockerfile b/pkg/configuration/roles/tendermint/files/Dockerfile new file mode 100644 index 0000000..b33d0f9 --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/Dockerfile @@ -0,0 +1,7 @@ +ARG tm_version=v0.31.5 +FROM tendermint/tendermint:${tm_version} +LABEL maintainer "contact@ipdb.global" +WORKDIR / +USER root +RUN apk --update add bash +ENTRYPOINT ["/usr/bin/tendermint"] diff --git a/pkg/configuration/roles/tendermint/files/access_pub_key.conf b/pkg/configuration/roles/tendermint/files/access_pub_key.conf new file mode 100644 index 0000000..5de7bb4 --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/access_pub_key.conf @@ -0,0 +1,10 @@ +# Serve the public key for a tendermint instance + +server { + listen 9986 default_server; + listen [::]:9986 default_server ipv6only=on; + location / { + root /usr/share/nginx/; + autoindex on; + } +} \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/files/config.toml b/pkg/configuration/roles/tendermint/files/config.toml new file mode 100644 index 0000000..c2ed679 --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/config.toml @@ -0,0 +1,169 @@ +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +##### main base config options ##### + +# TCP or UNIX socket address of the ABCI application, +# or the name of an ABCI application compiled in with the Tendermint binary +proxy_app = "tcp://127.0.0.1:26658" + +# A custom human readable name for this node +moniker = "d16137710ef8" + +# If this node is many blocks behind the tip of the chain, FastSync +# allows them to catchup quickly by downloading blocks in parallel +# and verifying their commits +fast_sync = true + +# Database backend: leveldb | memdb +db_backend = "leveldb" + +# Database directory +db_path = "data" + +# Output level for logging, including package level options +log_level = "main:info,state:info,*:error" + +##### additional base config options ##### + +# Path to the JSON file containing the initial validator set and other meta data +genesis_file = "config/genesis.json" + +# Path to the JSON file containing the private key to use as a validator in the consensus protocol +priv_validator_file = "config/priv_validator.json" + +# Path to the JSON file containing the private key to use for node authentication in the p2p protocol +node_key_file = "config/node_key.json" + +# Mechanism to connect to the ABCI application: socket | grpc +abci = "socket" + +# TCP or UNIX socket address for the profiling server to listen on +prof_laddr = "" + +# If true, query the ABCI app on connecting to a new peer +# so the app can decide if we should keep the connection or not +filter_peers = false + +##### advanced configuration options ##### + +##### rpc server configuration options ##### +[rpc] + +# TCP or UNIX socket address for the RPC server to listen on +laddr = "tcp://0.0.0.0:26657" + +# TCP or UNIX socket address for the gRPC server to listen on +# NOTE: This server only supports /broadcast_tx_commit +grpc_laddr = "" + +# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool +unsafe = false + +##### peer to peer configuration options ##### +[p2p] + +# Address to listen for incoming connections +laddr = "tcp://0.0.0.0:26656" + +# Comma separated list of seed nodes to connect to +seeds = "" + +# Comma separated list of nodes to keep persistent connections to +# Do not add private peers to this list if you don't want them advertised +persistent_peers = "" + +# Path to address book +addr_book_file = "config/addrbook.json" + +# Set true for strict address routability rules +addr_book_strict = true + +# Time to wait before flushing messages out on the connection, in ms +flush_throttle_timeout = 100 + +# Maximum number of peers to connect to +max_num_peers = 50 + +# Maximum size of a message packet payload, in bytes +max_packet_msg_payload_size = 1024 + +# Rate at which packets can be sent, in bytes/second +send_rate = 512000 + +# Rate at which packets can be received, in bytes/second +recv_rate = 512000 + +# Set true to enable the peer-exchange reactor +pex = true + +# Seed mode, in which node constantly crawls the network and looks for +# peers. If another node asks it for addresses, it responds and disconnects. +# +# Does not work if the peer-exchange reactor is disabled. +seed_mode = false + +# Authenticated encryption +auth_enc = true + +# Comma separated list of peer IDs to keep private (will not be gossiped to other peers) +private_peer_ids = "" + +##### mempool configuration options ##### +[mempool] + +recheck = true +recheck_empty = true +broadcast = true +wal_dir = "data/mempool.wal" + +##### consensus configuration options ##### +[consensus] + +wal_file = "data/cs.wal/wal" + +# All timeouts are in milliseconds +timeout_propose = 3000 +timeout_propose_delta = 500 +timeout_prevote = 1000 +timeout_prevote_delta = 500 +timeout_precommit = 1000 +timeout_precommit_delta = 500 +timeout_commit = 1000 + +# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) +skip_timeout_commit = false + +# BlockSize +max_block_size_txs = 10000 +max_block_size_bytes = 1 + +# EmptyBlocks mode and possible interval between empty blocks in seconds +create_empty_blocks = true +create_empty_blocks_interval = 0 + +# Reactor sleep duration parameters are in milliseconds +peer_gossip_sleep_duration = 100 +peer_query_maj23_sleep_duration = 2000 + +##### transactions indexer configuration options ##### +[tx_index] + +# What indexer to use for transactions +# +# Options: +# 1) "null" (default) +# 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). +indexer = "kv" + +# Comma-separated list of tags to index (by default the only tag is tx hash) +# +# It's recommended to index only a subset of tags due to possible memory +# bloat. This is, of course, depends on the indexer's DB and the volume of +# transactions. +index_tags = "" + +# When set to true, tells indexer to index all tags. Note this may be not +# desirable (see the comment above). IndexTags has a precedence over +# IndexAllTags (i.e. when given both, IndexTags will be indexed). +index_all_tags = false \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/files/genesis.json b/pkg/configuration/roles/tendermint/files/genesis.json new file mode 100644 index 0000000..9b7ea8f --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/genesis.json @@ -0,0 +1,6 @@ +{ + "genesis_time": "0001-01-01T00:00:00Z", + "chain_id": "test-chain-KPI1Ud", + "validators": [], + "app_hash": "" +} diff --git a/pkg/configuration/roles/tendermint/tasks/centos.yml b/pkg/configuration/roles/tendermint/tasks/centos.yml new file mode 100644 index 0000000..39d6331 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/centos.yml @@ -0,0 +1,16 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install dependencies | yum + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - unzip + - nginx + - jq + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/common.yml b/pkg/configuration/roles/tendermint/tasks/common.yml new file mode 100644 index 0000000..1db9b33 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/common.yml @@ -0,0 +1,57 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +- name: Download Tendermint Binary + get_url: + url: "{{ tendermint_binary_url }}" + dest: "{{ ansible_env.HOME }}/tendermint_binary.zip" + mode: 0755 + tags: [tendermint] + +- name: Extract Tendermint Binary + unarchive: + src: "{{ ansible_env.HOME }}/tendermint_binary.zip" + dest: /usr/local/bin + remote_src: yes + tags: [tendermint] + +- name: Creating Tendermint config directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + recurse: yes + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [tendermint] + +- name: Copying Tendermint config files + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: "{{ tendermint_conf_files }}" + tags: [tendermint] + +- name: Creating files for key exchange + file: + template: src=start_tendermint.j2 dest=/home/vagrant/test.sh + mode: 0755 + tags: [tendermint] + +- name: Start nginx to host public_key + systemd: name=nginx state=restarted + tags: [tendermint] + +- name: Tendermint Process Check + shell: pgrep tendermint | wc -l + register: tm_pchk + tags: [bigchaindb] + +- name: Run tendermint + shell: nohup /bin/bash /home/vagrant/test.sh > /tmp/tendermint_log_$(date +%Y%m%d_%H%M%S) 2>&1 & + register: run_tendermint + when: tm_pchk.stdout| int == 0 + failed_when: "'FAILED' in run_tendermint.stderr or run_tendermint.rc != 0" + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/debian.yml b/pkg/configuration/roles/tendermint/tasks/debian.yml new file mode 100644 index 0000000..61504e9 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/debian.yml @@ -0,0 +1,17 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install dependencies for Tendermint | apt + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - wget + - unzip + - nginx + - jq + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/fedora.yml b/pkg/configuration/roles/tendermint/tasks/fedora.yml new file mode 100644 index 0000000..c9ce062 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/fedora.yml @@ -0,0 +1,26 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Install dependencies | dnf + dnf: + name: "{{ item }}" + state: present + with_items: + - wget + - unzip + - libselinux-python + - policycoreutils-python + - nginx + - jq + tags: [tendermint] + +- name: Allow nginx to host public key + seport: + ports: 9986 + proto: tcp + setype: http_port_t + state: present + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/main.yml b/pkg/configuration/roles/tendermint/tasks/main.yml new file mode 100644 index 0000000..188164c --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/main.yml @@ -0,0 +1,24 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: (stack_type|lower == "docker" or stack_type|lower == "cloud") and info_result is failed + tags: [tendermint] + +- import_tasks: start.yml + when: action|lower == 'start' + +- import_tasks: stop.yml + when: action|lower == 'stop' \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/start.yml b/pkg/configuration/roles/tendermint/tasks/start.yml new file mode 100644 index 0000000..1869741 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/start.yml @@ -0,0 +1,90 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Building Tendermint Docker + docker_image: + name: "{{ tendermint_image_name }}" + state: build + tag: "{{tendermint_image_tag }}" + dockerfile: Dockerfile + path: "{{ home_dir }}/bigchaindb/pkg/configuration/roles/tendermint/files" + buildargs: + tm_version: "{{ tm_version }}" + nocache: yes + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Configuring Tendermint Containers + docker_container: + name: "tm_config_gen" + image: "{{ tendermint_image_name }}:{{ tendermint_image_tag }}" + detach: true + env: + STACK_SIZE: "{{ stack_size }}" + TM_DOCKER_NAME: "{{ tendermint_docker_name }}" + volumes: + - "{{ tendermint_host_mount_config_dir }}{{ tendermint_home }}:{{ tendermint_home }}" + - "../scripts:/scripts" + entrypoint: '' + command: /usr/bin/env bash -c "/scripts/tm_config_gen" + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Starting Tendermint Containers + docker_container: + name: "{{ tendermint_docker_name }}{{ item }}" + hostname: "{{ tendermint_docker_name }}{{ item }}" + image: "{{ tendermint_image_name }}:{{ tendermint_image_tag }}" + detach: true + network_mode: bridge + networks: + - name: "{{ bigchaindb_docker_net }}" + published_ports: + - "{{ tendermint_p2p_port }}" + - "{{ tendermint_rpc_port }}" + volumes: + - "{{ tendermint_host_mount_dir }}{{ item|string }}{{ tendermint_home }}:{{ tendermint_home }}" + - "{{ tendermint_host_mount_dir }}{{ item|string }}{{ tendermint_data }}:{{ tendermint_data }}" + - "{{ tendermint_host_mount_config_dir }}{{ tendermint_home }}:/tendermint_config" + - "../scripts:/scripts" + entrypoint: '' + command: bash -c "/scripts/tm_start" + env: + STACK_SIZE: "{{ stack_size|string }}" + TM_DOCKER_NAME: "{{ tendermint_docker_name|string }}" + TM_P2P_PORT: "{{ tendermint_p2p_port|string }}" + PLANETMINT_DOCKER_NAME: "{{ bigchaindb_docker_name|string }}" + _ITEM: "{{ item|string }}" + state: started + keep_volumes: true + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Creating directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [tendermint] + +- import_tasks: debian.yml + when: stack_type|lower == "local" and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [tendermint] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [tendermint] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (distribution_name == "fedora") + tags: [tendermint] + +- import_tasks: common.yml + when: stack_type|lower == "local" + tags: [tendermint] diff --git a/pkg/configuration/roles/tendermint/tasks/stop.yml b/pkg/configuration/roles/tendermint/tasks/stop.yml new file mode 100644 index 0000000..f12340b --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/stop.yml @@ -0,0 +1,62 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +--- +- name: Stopping Tendermint Containers I + docker_container: + name: "{{ tendermint_docker_name }}{{ item }}" + image: "tendermint/tendermint:{{ tm_version }}" + state: absent + keep_volumes: no + force_kill: yes + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Configuring Tendermint Containers II + docker_container: + name: "tm_config_gen" + image: "tendermint/tendermint:{{ tm_version }}" + detach: true + force_kill: yes + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Removing Tendermint Directories I + file: + path: "{{ tendermint_host_mount_dir }}{{ item }}" + state: absent + force: yes + with_sequence: start=0 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Removing Tendermint Directories II + file: + path: "{{ tendermint_host_mount_config_dir }}" + state: absent + force: yes + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Get Running Tendermint Process + shell: "ps aux | grep \"[t]endermint\" | awk '{print $2}'" + register: tendermint_ps + when: stack_type|lower == "local" + tags: [tendermint] + +- name: Kill Tendermint Process + shell: "kill {{ item }}" + with_items: "{{ tendermint_ps.stdout_lines }}" + when: stack_type|lower == "local" and tendermint_ps.stdout_lines|length > 0 + tags: [tendermint] + +- name: Removing Tendermint Directories + file: + path: "{{ tendermint_home }}" + state: absent + force: yes + when: stack_type|lower == "local" + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 b/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 new file mode 100644 index 0000000..75631f4 --- /dev/null +++ b/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 @@ -0,0 +1,98 @@ +#!/bin/bash +set -euo pipefail + +{%- set peers = [] -%} +{%- set validator_power = [] -%} +{% for host in hostvars %} +{{- peers.append(host) -}} +{{- validator_power.append(10) -}} +{% endfor %} + +tm_persistent_peers="{{ peers| reverse | join(',') }}" +tm_validators="{{ peers | reverse | join(',') }}" +tm_validator_power="{{ validator_power | join(',') }}" +tm_pub_key_access_port="9986" +tm_p2p_port="{{ tendermint_p2p_port|string }}" +tm_proxy_app="{{ ansible_hostname|string }}" +tm_abci_port="{{ tendermint_abci_port|string }}" +tm_instance_name="{{ ansible_hostname }}" + +{% raw %} +# Container vars +RETRIES=0 +CANNOT_INITIATLIZE_INSTANCE='Cannot start instance, if initial validator(s) are unreachable.' + +TM_GENESIS_FILE=/tendermint/config/genesis.json +TM_PUB_KEY_DIR=/usr/share/nginx + +if [ ! -f /tendermint/config/priv_validator.json ]; then + /usr/local/bin/tendermint gen_validator > /tendermint/config/priv_validator.json + # pub_key.json will be served by the nginx container + cat /tendermint/config/priv_validator.json + cat /tendermint/config/priv_validator.json | jq ".pub_key" > "$TM_PUB_KEY_DIR"/pub_key.json +fi + +if [ ! -f /tendermint/config/node_key.json ]; then + /usr/local/bin/tendermint --home "/tendermint" gen_node_key > "$TM_PUB_KEY_DIR"/address +fi + +# fill genesis file with validators +IFS=',' read -ra VALS_ARR <<< "$tm_validators" +IFS=',' read -ra VAL_POWERS_ARR <<< "$tm_validator_power" +if [ ${#VALS_ARR[@]} -ne ${#VAL_POWERS_ARR[@]} ]; then + echo "Invalid configuration of Validator(s) and Validator Power(s)" + exit 1 +fi +for i in "${!VALS_ARR[@]}"; do + # wait until validator generates priv/pub key pair + set +e + echo Validator: "${VALS_ARR[$i]}" + echo Validator Power: "${VAL_POWERS_ARR[$i]}" + echo "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + RETRIES=$((RETRIES+1)) + if [ $RETRIES -eq 10 ]; then + echo "${CANNOT_INITIATLIZE_INSTANCE}" + exit 1 + fi + # 300(30 * 10(retries)) second timeout before container dies if it cannot find initial peers + sleep 30 + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + echo "Cannot connect to Tendermint instance: ${VALS_ARR[$i]}" + done + set -e + # add validator to genesis file along with its pub_key + curl -s "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" | jq ". as \$k | {pub_key: \$k, power: \"${VAL_POWERS_ARR[$i]}\", name: \"${VALS_ARR[$i]}\"}" > pub_validator.json + cat /tendermint/config/genesis.json | jq ".validators |= .+ [$(cat pub_validator.json)]" > tmpgenesis && mv tmpgenesis /tendermint/config/genesis.json + rm pub_validator.json + done +# construct persistent peers +IFS=',' read -ra PEERS_ARR <<< "$tm_persistent_peers" +peers=() +for s in "${PEERS_ARR[@]}"; do + echo "http://$s:$tm_pub_key_access_port/address" + curl -s --fail "http://$s:$tm_pub_key_access_port/address" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + RETRIES=$((RETRIES+1)) + if [ $RETRIES -eq 10 ]; then + echo "${CANNOT_INITIATLIZE_INSTANCE}" + exit 1 + fi + # 300(30 * 10(retries)) second timeout before container dies if it cannot find initial peers + sleep 30 + curl -s --fail "http://$s:$tm_pub_key_access_port/address" > /dev/null + ERR=$? + echo "Cannot get address for Tendermint instance: ${s}" + done + peer_addr=$(curl -s "http://$s:$tm_pub_key_access_port/address") + peers+=("$peer_addr@$s:$tm_p2p_port") +done +peers=$(IFS=','; echo "${peers[*]}") + +echo "INFO: starting tendermint..." +/usr/local/bin/tendermint node --home "/tendermint" --p2p.persistent_peers="$peers" --proxy_app="tcp://0.0.0.0:26658" --consensus.create_empty_blocks=false --p2p.pex=false +{% endraw %} diff --git a/pkg/configuration/vars/stack-config.yml b/pkg/configuration/vars/stack-config.yml new file mode 100644 index 0000000..e69de29 diff --git a/pkg/scripts/Vagrantfile b/pkg/scripts/Vagrantfile new file mode 100644 index 0000000..a0962f0 --- /dev/null +++ b/pkg/scripts/Vagrantfile @@ -0,0 +1,102 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Required modules +require 'yaml' +VAGRANTFILE_API_VERSION = "2" + +Vagrant.require_version ">= 1.8.7" +# Validate if all the required plugins are present +# vagrant-hostmanager replaced +required_plugins = ["vagrant-cachier", "vagrant-vbguest", "vagrant-hosts", "vagrant-azure"] +required_plugins.each do |plugin| + if not Vagrant.has_plugin?(plugin) + raise "Required vagrant plugin #{plugin} not found. Please run `vagrant plugin install #{plugin}`" + end +end + +# Configuration files +CONFIGURATION_FILE = 'bigchaindb/pkg/configuration/vars/stack-config.yml' +HOSTS_FILE = 'bigchaindb/pkg/configuration/hosts/all' +HOST_VARS_PATH = 'bigchaindb/pkg/configuration/host_vars' + +# Read configuration file(s) +instances_config = YAML.load_file(File.join(File.dirname(__FILE__), CONFIGURATION_FILE)) +hosts_config = File.open(HOSTS_FILE, 'w+') + +# Vars needed for VM configuration +if (instances_config["stack_type"] == "cloud" && instances_config["stack_type_provider"] == "azure") + box_name = "azure-dummy" +else + box_name = instances_config['stack_box_name'] +end + +# configure instance names and private ip addresses + +instances_arr = Array.new +private_ipam_arr = Array.new +if instances_config['stack_type'] == "local" + for i in 1..Integer(instances_config['stack_size']) + instance_name = "bdb-node-#{i}" + instance_ip_address = "10.20.30.#{i+10}" + instances_arr.push instance_name + private_ipam_arr.push instance_ip_address + hosts_config.puts("#{instance_ip_address} ansible_user=vagrant") + File.open("#{HOST_VARS_PATH}/#{instance_ip_address}", "w+") {|f| \ + f.write("ansible_ssh_private_key_file: .vagrant/machines/#{instance_name}/virtualbox/private_key") } + end +elsif + instance_name = instances_config["azure_dns_prefix"] + "." + instances_config["azure_region"] + ".cloudapp.azure.com" + instances_arr.push(instances_config["azure_dns_prefix"]) + hosts_config.puts("#{instance_name} ansible_user=#{instances_config["azure_admin_username"]}") + File.open("#{HOST_VARS_PATH}/#{instance_name}", "w+") {|f| \ + f.write("ansible_ssh_private_key_file: #{instances_config["ssh_private_key_path"]}") } +end +hosts_config.close + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + instances_arr.each_with_index do |instance, index| + config.vm.define "#{instance}" do |node| + node.vm.box = box_name + if instances_config["stack_type"] == "cloud" + node.ssh.private_key_path= instances_config["ssh_private_key_path"] + end + node.vm.box_check_update = false + # Workaround until vagrant cachier plugin supports dnf + if !(box_name.include? "fedora") + if Vagrant.has_plugin?("vagrant-cachier") + node.cache.scope = :box + end + elsif box_name == "ubuntu/xenial64" + if Vagrant.has_plugin?("vagrant-vbguest") + node.vbguest.auto_update = true + node.vbguest.auto_reboot = true + config.vbguest.no_install = true + config.vbguest.no_remote = true + end + end + node.vm.synced_folder "bigchaindb", "/opt/stack/bigchaindb" + node.vm.hostname = instance + node.vm.provision :hosts, :sync_hosts => true + node.ssh.insert_key = true + node.vm.network :private_network, ip: private_ipam_arr[index] + node.vm.provider :virtualbox do |vb, override| + vb.customize ["modifyvm", :id, "--memory", instances_config['stack_vm_memory'].to_s] + vb.customize ["modifyvm", :id, "--cpus", instances_config['stack_vm_cpus'].to_s] + end + node.vm.provider :azure do |azure, override| + azure.tenant_id = ENV['AZURE_TENANT_ID'] + azure.client_id = ENV['AZURE_CLIENT_ID'] + azure.client_secret = ENV['AZURE_CLIENT_SECRET'] + azure.subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] + azure.admin_username = instances_config["azure_admin_username"] + azure.dns_name = instances_config["azure_dns_prefix"] + azure.vm_name = instances_config["azure_dns_prefix"] + azure.vm_size = instances_config["azure_vm_size"] + azure.vm_image_urn = instances_config["azure_image_urn"] + azure.resource_group_name = instances_config["azure_resource_group"] + azure.location = instances_config["azure_region"] + end + end + end +end \ No newline at end of file diff --git a/pkg/scripts/all-in-one.bash b/pkg/scripts/all-in-one.bash new file mode 100755 index 0000000..2d9565c --- /dev/null +++ b/pkg/scripts/all-in-one.bash @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +# MongoDB configuration +[ "$(stat -c %U /data/db)" = mongodb ] || chown -R mongodb /data/db + +# Planetmint configuration +bigchaindb-monit-config + +nohup mongod --bind_ip_all > "$HOME/.bigchaindb-monit/logs/mongodb_log_$(date +%Y%m%d_%H%M%S)" 2>&1 & + +# Tendermint configuration +tendermint init + +monit -d 5 -I -B diff --git a/pkg/scripts/bigchaindb-monit-config b/pkg/scripts/bigchaindb-monit-config new file mode 100644 index 0000000..bec155c --- /dev/null +++ b/pkg/scripts/bigchaindb-monit-config @@ -0,0 +1,199 @@ +#!/bin/bash + +set -o nounset + +# Check if directory for monit logs exists +if [ ! -d "$HOME/.bigchaindb-monit" ]; then + mkdir -p "$HOME/.bigchaindb-monit" +fi + +monit_pid_path=${MONIT_PID_PATH:=$HOME/.bigchaindb-monit/monit_processes} +monit_script_path=${MONIT_SCRIPT_PATH:=$HOME/.bigchaindb-monit/monit_script} +monit_log_path=${MONIT_LOG_PATH:=$HOME/.bigchaindb-monit/logs} +monitrc_path=${MONITRC_PATH:=$HOME/.monitrc} + +function usage() { + cat <${monit_script_path} < /dev/null 2>&1 & + + echo \$! > \$2 + popd + + ;; + + stop_bigchaindb) + + kill -2 \`cat \$2\` + rm -f \$2 + + ;; + + start_tendermint) + + pushd \$4 + + nohup tendermint node >> \$3/tendermint.out.log 2>> \$3/tendermint.err.log & + + echo \$! > \$2 + popd + + ;; + + stop_tendermint) + + kill -2 \`cat \$2\` + rm -f \$2 + + ;; + +esac +exit 0 +EOF +chmod +x ${monit_script_path} + +cat >${monit_script_path}_logrotate <${monitrc_path} < 200 MB then + exec "${monit_script_path}_logrotate rotate_tendermint_logs ${monit_log_path}/tendermint.out.log $monit_pid_path/tendermint.pid" + +check file tendermint.err.log with path ${monit_log_path}/tendermint.err.log + if size > 200 MB then + exec "${monit_script_path}_logrotate rotate_tendermint_logs ${monit_log_path}/tendermint.err.log $monit_pid_path/tendermint.pid" + +EOF + +# Setting permissions for control file +chmod 0700 ${monitrc_path} + +echo -e "Planetmint process manager configured!" +set -o errexit diff --git a/pkg/scripts/bootstrap.sh b/pkg/scripts/bootstrap.sh new file mode 100755 index 0000000..58ffdfe --- /dev/null +++ b/pkg/scripts/bootstrap.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +BASEDIR="${BASH_SOURCE%/*}" +if [[ ! -d "$BASEDIR" ]]; then BASEDIR="$PWD"; fi +. "$BASEDIR/bootstrap_constants.sh" +. "$BASEDIR/bootstrap_helper.sh" + +# OS ID(centos, debian, fedora, ubuntu) +OS="" +# OS Version(7, 9, 24, 16.04) +VER="" +# OP (install, uninstall) +OPERATION=${OPERATION:=""} + +# Parsing arguments +while [[ $# -gt 1 ]]; do + arg="$1" + case $arg in + --operation) + OPERATION="$2" + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift +done + +# sanity checks +if [[ -z "${OPERATION:?Missing '--operation' [install,uninstall])}" ]] ; then + exit 1 +fi + +validate_os_configuration(){ + valid_os=1 + if [ -f $1 ]; then + . $1 + OS=$ID + VER=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + OS=$(lsb_release -si) + VER=$(lsb_release -sr) + elif [ "$(uname -s)" == "Darwin" ]; then + echo "Using macOS" + OS="macOS" + VER="None" + valid_os=True + return + else + echo "Cannot find $OS_CONF. Pass arguments to your OS configurations: NAME, VERSION_ID. + Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + exit 1 + fi + for os in "${SUPPORTED_OS[@]}"; do + if [[ $os = $OS ]]; then + valid_os=true + break + fi + done +} + +validate_os_configuration $OS_CONF +echo "Operation Sytem: $OS" +echo "Version: $VER" +# Installing dependencies +if [ "$OPERATION" = "install" ]; then + install_deps=$(validate_os_version_and_deps true $OS $VER) + if [[ $install_deps -eq 1 ]]; then + for dep in "${OS_DEPENDENCIES[@]}" + do + install_"$dep" $OS + done + elif [[ $install_deps -eq 2 ]]; then + echo "Unsupported $OS Version: $VER" + else + echo "Dependencies already installed:[ ${OS_DEPENDENCIES[*]} ]" + fi +# Uninstalling dependencies +elif [ "$OPERATION" = "uninstall" ]; then + uninstall_deps=$(validate_os_version_and_deps true $OS $VER) + if [[ $install_deps -eq 1 ]]; then + echo "Dependencies already uninstalled:[ ${OS_DEPENDENCIES[*]} ]" + elif [[ $install_deps -eq 2 ]]; then + echo "Unsupported $OS Version: $VER" + else + for dep in "${OS_DEPENDENCIES[@]}" + do + uninstall_"$dep" $OS + done + fi +else + echo "Invalid Operation specified. Only [install, uninstall] are supported." + exit 1 +fi diff --git a/pkg/scripts/bootstrap_constants.sh b/pkg/scripts/bootstrap_constants.sh new file mode 100755 index 0000000..3e8b07f --- /dev/null +++ b/pkg/scripts/bootstrap_constants.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +OS_CONF=/etc/os-release +declare -a SUPPORTED_OS=('centos' 'fedora' 'ubuntu' 'debian' 'macOS') +declare -a OS_DEPENDENCIES=('ansible') +MINIMUM_UBUNTU_VERSION=16.04 +MINIMUM_CENTOS_VERSION=7 +MINIMUM_FEDORA_VERSION=24 +MINIMUM_DEBIAN_VERSION=8 diff --git a/pkg/scripts/bootstrap_helper.sh b/pkg/scripts/bootstrap_helper.sh new file mode 100755 index 0000000..b299e18 --- /dev/null +++ b/pkg/scripts/bootstrap_helper.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +BASEDIR="${BASH_SOURCE%/*}" +if [[ ! -d "$BASEDIR" ]]; then BASEDIR="$PWD"; fi +. "$BASEDIR/bootstrap_constants.sh" + +validate_os_version_and_deps(){ + if $1; then + case $2 in + centos) + if [[ ($(version_compare_gt $3 $MINIMUM_CENTOS_VERSION) == 0) + || ($(version_compare_eq $3 $MINIMUM_CENTOS_VERSION) == 0) ]]; then + rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo 2 + fi + ;; + debian) + if [[ ($(version_compare_gt $3 $MINIMUM_DEBIAN_VERSION) == 0) + || ($(version_compare_eq $3 $MINIMUM_DEBIAN_VERSION) == 0) ]]; then + dpkg -s "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo 2 + fi + ;; + fedora) + if [[ ($(version_compare_gt $3 $MINIMUM_FEDORA_VERSION) == 0) + || ($(version_compare_eq $3 $MINIMUM_FEDORA_VERSION) == 0) ]]; then + rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo 2 + fi + ;; + ubuntu) + if [[ ($(version_compare_gt $3 $MINIMUM_UBUNTU_VERSION) == 0) + || ($(version_compare_eq $3 $MINIMUM_UBUNTU_VERSION) == 0) ]]; then + dpkg -s "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo 2 + fi + ;; + macOS) + pip show "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + ;; + *) + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + exit 1 + ;; + esac + else + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + fi +} + +version_compare_gt(){ + test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1" + echo $? +} + +version_compare_eq(){ + test "$(printf '%s\n' "$@" | sort -V | head -n 1)" == "$2" + echo $? +} + + +install_ansible() { + echo "Installing Ansible..." + case $1 in + centos) + yum install epel-release -y + yum install -y https://centos7.iuscommunity.org/ius-release.rpm + yum install ansible -y + ;; + debian) + apt-get update -y && apt-get install --fix-missing + apt-get install lsb-release software-properties-common gnupg -y + echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/ansible-debian.list + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + apt-get update + apt-get install -y ansible + echo 'localhost' > /etc/ansible/hosts + ;; + fedora) + export LC_ALL=C + dnf makecache + dnf -y install ansible + ;; + macOS) + easy_install pip + pip install ansible + ;; + ubuntu) + apt-get update -y + apt-get install -y software-properties-common + apt-add-repository ppa:ansible/ansible -y + apt-get update -y + apt-get install -y ansible + ;; + *) + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + esac +} + +uninstall_ansible() { + echo "Uninstalling Ansible..." + case $1 in + centos) + yum remove ansible -y + ;; + debian) + apt-get purge ansible -y + ;; + fedora) + export LC_ALL=C + dnf remove ansible -y + ;; + macOS) + pip uninstall ansible -y + ;; + ubuntu) + apt-get purge ansible -y + ;; + *) + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + esac +} diff --git a/pkg/scripts/functions-common b/pkg/scripts/functions-common new file mode 100755 index 0000000..8cfc055 --- /dev/null +++ b/pkg/scripts/functions-common @@ -0,0 +1,360 @@ +#!/bin/bash + +# Save trace setting +_XTRACE_FUNCTIONS_COMMON=$(set +o | grep xtrace) +set +o xtrace + +# Distro Functions +# ================ + +# Determine OS Vendor, Release and Update + +# +# NOTE : For portability, you almost certainly do not want to use +# these variables directly! The "is_*" functions defined below this +# bundle up compatible platforms under larger umbrellas that we have +# determinted are compatible enough (e.g. is_ubuntu covers Ubuntu & +# Debian, is_fedora covers RPM-based distros). Higher-level functions +# such as "install_package" further abstract things in better ways. +# +# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc +# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora) +# ``os_PACKAGE`` - package type: ``deb`` or ``rpm`` +# ``os_CODENAME`` - vendor's codename for release: ``xenial`` + +#declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME + +# Make a *best effort* attempt to install lsb_release packages for the +# user if not available. Note can't use generic install_package* +# because they depend on this! +function _ensure_lsb_release { + if [[ -x $(command -v lsb_release 2>/dev/null) ]]; then + return + fi + + if [[ -x $(command -v apt-get 2>/dev/null) ]]; then + sudo apt-get install -y lsb-release + elif [[ -x $(command -v zypper 2>/dev/null) ]]; then + sudo zypper -n install lsb-release + elif [[ -x $(command -v dnf 2>/dev/null) ]]; then + sudo dnf install -y redhat-lsb-core + elif [[ -x $(command -v yum 2>/dev/null) ]]; then + # all rh patforms (fedora, centos, rhel) have this pkg + sudo yum install -y redhat-lsb-core + else + die $LINENO "Unable to find or auto-install lsb_release" + fi +} + +# GetOSVersion +# Set the following variables: +# - os_RELEASE +# - os_CODENAME +# - os_VENDOR +# - os_PACKAGE +function GetOSVersion { + # We only support distros that provide a sane lsb_release + _ensure_lsb_release + + os_RELEASE=$(lsb_release -r -s) + os_CODENAME=$(lsb_release -c -s) + os_VENDOR=$(lsb_release -i -s) + + if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then + os_PACKAGE="deb" + else + os_PACKAGE="rpm" + fi + + typeset -xr os_VENDOR + typeset -xr os_RELEASE + typeset -xr os_PACKAGE + typeset -xr os_CODENAME +} + +# Translate the OS version values into common nomenclature +# Sets global ``DISTRO`` from the ``os_*`` values +#declare -g DISTRO + +function GetDistro { + GetOSVersion + if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \ + "$os_VENDOR" =~ (LinuxMint) ]]; then + # 'Everyone' refers to Ubuntu / Debian / Mint releases by + # the code name adjective + DISTRO=$os_CODENAME + elif [[ "$os_VENDOR" =~ (Fedora) ]]; then + # For Fedora, just use 'f' and the release + DISTRO="f$os_RELEASE" + elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then + DISTRO="opensuse-$os_RELEASE" + elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then + # just use major release + DISTRO="sle${os_RELEASE%.*}" + elif [[ "$os_VENDOR" =~ (Red.*Hat) || \ + "$os_VENDOR" =~ (CentOS) || \ + "$os_VENDOR" =~ (Scientific) || \ + "$os_VENDOR" =~ (OracleServer) || \ + "$os_VENDOR" =~ (Virtuozzo) ]]; then + # Drop the . release as we assume it's compatible + # XXX re-evaluate when we get RHEL10 + DISTRO="rhel${os_RELEASE::1}" + elif [[ "$os_VENDOR" =~ (XenServer) ]]; then + DISTRO="xs${os_RELEASE%.*}" + elif [[ "$os_VENDOR" =~ (kvmibm) ]]; then + DISTRO="${os_VENDOR}${os_RELEASE::1}" + else + die $LINENO "Unable to determine DISTRO, can not continue." + fi + typeset -xr DISTRO +} + +# Utility function for checking machine architecture +# is_arch arch-type +function is_arch { + [[ "$(uname -m)" == "$1" ]] +} + + +# Determine if current distribution is a Fedora-based distribution +# (Fedora, RHEL, CentOS, etc). +# is_fedora +function is_fedora { + if [[ -z "$os_VENDOR" ]]; then + GetOSVersion + fi + + [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \ + [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \ + [ "$os_VENDOR" = "CentOS" ] +} + + +# Determine if current distribution is a SUSE-based distribution +# (openSUSE, SLE). +# is_suse +function is_suse { + if [[ -z "$os_VENDOR" ]]; then + GetOSVersion + fi + + [[ "$os_VENDOR" =~ (openSUSE) || "$os_VENDOR" == "SUSE LINUX" ]] +} + + +# Determine if current distribution is an Ubuntu-based distribution +# It will also detect non-Ubuntu but Debian-based distros +# is_ubuntu +function is_ubuntu { + if [[ -z "$os_PACKAGE" ]]; then + GetOSVersion + fi + [ "$os_PACKAGE" = "deb" ] +} + +# Package Functions +# ================= + +# Wrapper for ``apt-get update`` to try multiple times on the update +# to address bad package mirrors (which happen all the time). +function apt_get_update { + # only do this once per run + if [[ "$REPOS_UPDATED" == "True" && "$RETRY_UPDATE" != "True" ]]; then + return + fi + + # bail if we are offline + [[ "$OFFLINE" = "True" ]] && return + + local sudo="sudo" + [[ "$(id -u)" = "0" ]] && sudo="env" + + # time all the apt operations + time_start "apt-get-update" + + local proxies="http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} " + local update_cmd="$sudo $proxies apt-get update" + if ! timeout 300 sh -c "while ! $update_cmd; do sleep 30; done"; then + die $LINENO "Failed to update apt repos, we're dead now" + fi + + REPOS_UPDATED=True + # stop the clock + time_stop "apt-get-update" +} + +# Wrapper for ``apt-get`` to set cache and proxy environment variables +# Uses globals ``OFFLINE``, ``*_proxy`` +# apt_get operation package [package ...] +function apt_get { + local xtrace result + xtrace=$(set +o | grep xtrace) + set +o xtrace + + [[ "$OFFLINE" = "True" || -z "$@" ]] && return + local sudo="sudo" + [[ "$(id -u)" = "0" ]] && sudo="env" + + # time all the apt operations + time_start "apt-get" + + $xtrace + + $sudo DEBIAN_FRONTEND=noninteractive \ + http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \ + no_proxy=${no_proxy:-} \ + apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@" < /dev/null + result=$? + + # stop the clock + time_stop "apt-get" + return $result +} + + +# Distro-agnostic package installer +# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE`` +# install_package package [package ...] +function update_package_repo { + NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False} + REPOS_UPDATED=${REPOS_UPDATED:-False} + RETRY_UPDATE=${RETRY_UPDATE:-False} + + if [[ "$NO_UPDATE_REPOS" = "True" ]]; then + return 0 + fi + + if is_ubuntu; then + apt_get_update + fi +} + +function real_install_package { + if is_ubuntu; then + apt_get install "$@" + elif is_fedora; then + yum_install "$@" + elif is_suse; then + zypper_install "$@" + else + exit_distro_not_supported "installing packages" + fi +} + +# Distro-agnostic package installer +# install_package package [package ...] +function install_package { + update_package_repo + if ! real_install_package "$@"; then + RETRY_UPDATE=True update_package_repo && real_install_package "$@" + fi +} + +# Distro-agnostic function to tell if a package is installed +# is_package_installed package [package ...] +function is_package_installed { + if [[ -z "$@" ]]; then + return 1 + fi + + if [[ -z "$os_PACKAGE" ]]; then + GetOSVersion + fi + + if [[ "$os_PACKAGE" = "deb" ]]; then + dpkg -s "$@" > /dev/null 2> /dev/null + elif [[ "$os_PACKAGE" = "rpm" ]]; then + rpm --quiet -q "$@" + else + exit_distro_not_supported "finding if a package is installed" + fi +} + +# Distro-agnostic package uninstaller +# uninstall_package package [package ...] +function uninstall_package { + if is_ubuntu; then + apt_get purge "$@" + elif is_fedora; then + sudo ${YUM:-yum} remove -y "$@" ||: + elif is_suse; then + sudo zypper remove -y "$@" ||: + else + exit_distro_not_supported "uninstalling packages" + fi +} + +# Wrapper for ``yum`` to set proxy environment variables +# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM`` +# yum_install package [package ...] +function yum_install { + local result parse_yum_result + time_start "yum_install" + + # This is a bit tricky, because yum -y assumes missing or failed + # packages are OK (see [1]). We want devstack to stop if we are + # installing missing packages. + # + # Thus we manually match on the output (stack.sh runs in a fixed + # locale, so lang shouldn't change). + # + # If yum returns !0, we echo the result as "YUM_FAILED" and return + # that from the awk (we're subverting -e with this trick). + # Otherwise we use awk to look for failure strings and return "2" + # to indicate a terminal failure. + # + # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567 + parse_yum_result=' \ + BEGIN { result=0 } \ + /^YUM_FAILED/ { result=$2 } \ + /^No package/ { result=2 } \ + /^Failed:/ { result=2 } \ + //{ print } \ + END { exit result }' + (sudo_with_proxies "${YUM:-yum}" install -y "$@" 2>&1 || echo YUM_FAILED $?) \ + | awk "$parse_yum_result" && result=$? || result=$? + + time_stop "yum_install" + + # if we return 1, then the wrapper functions will run an update + # and try installing the package again as a defense against bad + # mirrors. This can hide failures, especially when we have + # packages that are in the "Failed:" section because their rpm + # install scripts failed to run correctly (in this case, the + # package looks installed, so when the retry happens we just think + # the package is OK, and incorrectly continue on). + if [ "$result" == 2 ]; then + die "Detected fatal package install failure" + fi + + return "$result" +} + +# zypper wrapper to set arguments correctly +# Uses globals ``OFFLINE``, ``*_proxy`` +# zypper_install package [package ...] +function zypper_install { + local sudo="sudo" + [[ "$(id -u)" = "0" ]] && sudo="env" + $sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \ + no_proxy="${no_proxy:-}" \ + zypper --non-interactive install --auto-agree-with-licenses "$@" +} + +function install_tendermint_bin { + wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip + unzip tendermint_${TM_VERSION}_linux_amd64.zip + sudo mv tendermint /usr/local/bin +} + +# Find out if a process exists by partial name. +# is_running name +function is_running { + local name=$1 + ps auxw | grep -v grep | grep ${name} > /dev/null + local exitcode=$? + return $exitcode +} + +# Restore xtrace +$_XTRACE_FUNCTIONS_COMMON \ No newline at end of file diff --git a/pkg/scripts/stack.sh b/pkg/scripts/stack.sh new file mode 100755 index 0000000..e95291b --- /dev/null +++ b/pkg/scripts/stack.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -o nounset + +# Make sure umask is sane +umask 022 + +# defaults +stack_branch=${STACK_BRANCH:="master"} +stack_repo=${STACK_REPO:="bigchaindb/bigchaindb"} +stack_size=${STACK_SIZE:=4} +stack_type=${STACK_TYPE:="docker"} +stack_type_provider=${STACK_TYPE_PROVIDER:=""} +# NOTE versions prior v0.28.0 have different priv_validator format! +tm_version=${TM_VERSION:="v0.31.5"} +mongo_version=${MONGO_VERSION:="3.6"} +stack_vm_memory=${STACK_VM_MEMORY:=2048} +stack_vm_cpus=${STACK_VM_CPUS:=2} +stack_box_name=${STACK_BOX_NAME:="ubuntu/xenial64"} +azure_subscription_id=${AZURE_SUBSCRIPTION_ID:=""} +azure_tenant_id=${AZURE_TENANT_ID:=""} +azure_client_secret=${AZURE_CLIENT_SECRET:=""} +azure_client_id=${AZURE_CLIENT_ID:=""} +azure_region=${AZURE_REGION:="westeurope"} +azure_image_urn=${AZURE_IMAGE_URN:="Canonical:UbuntuServer:16.04-LTS:latest"} +azure_resource_group=${AZURE_RESOURCE_GROUP:="bdb-vagrant-rg-$(date '+%Y-%m-%d')"} +azure_dns_prefix=${AZURE_DNS_PREFIX:="bdb-instance-$(date '+%Y-%m-%d')"} +azure_admin_username=${AZURE_ADMIN_USERNAME:="vagrant"} +azure_vm_size=${AZURE_VM_SIZE:="Standard_D2_v2"} +ssh_private_key_path=${SSH_PRIVATE_KEY_PATH:=""} + + +# Check for uninitialized variables +NOUNSET=${NOUNSET:-} +if [[ -n "$NOUNSET" ]]; then + set -o nounset +fi + +TOP_DIR=$(cd $(dirname "$0") && pwd) +SCRIPTS_DIR=$TOP_DIR/bigchaindb/pkg/scripts +CONF_DIR=$TOP_DIR/bigchaindb/pkg/configuration + + +function usage() { + cat < >(tee $log_file) 2>&1 +echo "Capturing output to $log_file" +echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')" + +function finish() { + echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')" +} +trap finish EXIT + +export STACK_REPO=$stack_repo +export STACK_BRANCH=$stack_branch +echo "Using bigchaindb repo: '$STACK_REPO'" +echo "Using bigchaindb branch '$STACK_BRANCH'" + +git clone https://github.com/${stack_repo}.git -b $stack_branch || true + +# Source utility functions +source ${SCRIPTS_DIR}/functions-common + +if [[ $stack_type == "local" ]]; then + mongo_version=$(echo "$mongo_version" | cut -d. -f-2) +fi + +# configure stack-config.yml +cat >$TOP_DIR/bigchaindb/pkg/configuration/vars/stack-config.yml <$CONF_DIR/hosts/all < /tendermint/config/genesis.tmp + +mv /tendermint/config/genesis.tmp /tendermint/config/genesis.json +rm /tendermint/config/node_key.json + +for i in $(seq $STACK_SIZE); do + + tendermint gen_validator > /tendermint/config/priv_validator$i.json; + tendermint gen_node_key > /tendermint/config/node_id$i; + cat tendermint/config/priv_validator$i.json \ + | jq ".Key.pub_key" \ + | jq ". as \$k | {pub_key: \$k, \ + power: \"10\", \ + name: \"$TM_DOCKER_NAME$i\"}" \ + > pub_validator$i.json; + + # added + cat tendermint/config/priv_validator$i.json \ + | jq ".Key" > tendermint/config/priv_validator_key$1.json + + cat /tendermint/config/genesis.json \ + | jq ".validators |= .+ [$(cat pub_validator$i.json)]" \ + > tmpgenesis; + + mv tmpgenesis /tendermint/config/genesis.json; +done diff --git a/pkg/scripts/tm_start b/pkg/scripts/tm_start new file mode 100755 index 0000000..7aa1465 --- /dev/null +++ b/pkg/scripts/tm_start @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Comment: This script is was carved out of start.yml task command. It's +# purpose is to copy generated in tm_config_gen configuration files from mounted +# volume, compile `--p2p.persistent_peers' cmd argument and start tendermint +# node. + +# NOTE following environment have to be set! +# $_ITEM -- stack size position identifier +# $STACK_SIZE -- self explanatory +# $TM_DOCKER_NAME -- used to identify tendermint containers in the network +# $PLANETMINT_DOCKER_NAME -- self explanatory +# $TM_P2P_PORT -- self explanatory + +# Copy confguration files from mounted config volume +cp /tendermint_config/genesis.json \ + /tendermint/config/genesis.json +cp /tendermint_config/priv_validator_key$_ITEM.json \ + /tendermint/config/priv_validator_key.json +cp /tendermint_config/node_key$_ITEM.json \ + /tendermint/config/node_key.json + +# Create peers array (to be passed to `tendermint node' command +peers=() +for i in $(seq $STACK_SIZE); do + peers+=($(cat /tendermint_config/node_id$i)@"$TM_DOCKER_NAME$i:$TM_P2P_PORT"); +done +peers=$(IFS=","; echo "${peers[*]}") + +echo "starting node with persistent peers set to:" +echo $peers +tendermint node \ + --p2p.persistent_peers="$peers" \ + --p2p.laddr "tcp://"$TM_DOCKER_NAME$_ITEM":26656" \ + --proxy_app="tcp://"$PLANETMINT_DOCKER_NAME$_ITEM":26658" \ + --consensus.create_empty_blocks=false \ + --p2p.pex=false diff --git a/pkg/scripts/unstack.sh b/pkg/scripts/unstack.sh new file mode 100755 index 0000000..fcda8f1 --- /dev/null +++ b/pkg/scripts/unstack.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +set -o nounset + +# Make sure umask is sane +umask 022 + +# defaults +stack_branch=${STACK_BRANCH:="master"} +stack_repo=${STACK_REPO:="bigchaindb/bigchaindb"} +stack_size=${STACK_SIZE:=4} +stack_type=${STACK_TYPE:="docker"} +stack_type_provider=${STACK_TYPE_PROVIDER:=""} +tm_version=${TM_VERSION:="0.31.5"} +mongo_version=${MONGO_VERSION:="3.6"} +stack_vm_memory=${STACK_VM_MEMORY:=2048} +stack_vm_cpus=${STACK_VM_CPUS:=2} +stack_box_name=${STACK_BOX_NAME:="ubuntu/xenial64"} +azure_subscription_id=${AZURE_SUBSCRIPTION_ID:=""} +azure_tenant_id=${AZURE_TENANT_ID:=""} +azure_client_secret=${AZURE_CLIENT_SECRET:=""} +azure_client_id=${AZURE_CLIENT_ID:=""} +azure_region=${AZURE_REGION:="westeurope"} +azure_image_urn=${AZURE_IMAGE_URN:="Canonical:UbuntuServer:16.04-LTS:latest"} +azure_resource_group=${AZURE_RESOURCE_GROUP:="bdb-vagrant-rg-$(date '+%Y-%m-%d')"} +azure_dns_prefix=${AZURE_DNS_PREFIX:="bdb-instance-$(date '+%Y-%m-%d')"} +azure_admin_username=${AZURE_ADMIN_USERNAME:="vagrant"} +azure_vm_size=${AZURE_VM_SIZE:="Standard_D2_v2"} +ssh_private_key_path=${SSH_PRIVATE_KEY_PATH:=""} +unstack_type=${UNSTACK_TYPE:="hard"} + + +# Check for uninitialized variables +NOUNSET=${NOUNSET:-} +if [[ -n "$NOUNSET" ]]; then + set -o nounset +fi + +TOP_DIR=$(cd $(dirname "$0") && pwd) +SCRIPTS_DIR=$TOP_DIR/bigchaindb/pkg/scripts +CONF_DIR=$TOP_DIR/bigchaindb/pkg/configuration + +function usage() { + cat < >(tee $log_file) 2>&1 +echo "Capturing output to $log_file" +echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')" + +function finish() { + echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')" +} +trap finish EXIT + +export STACK_REPO=$stack_repo +export STACK_BRANCH=$stack_branch +echo "Using bigchaindb repo: '$STACK_REPO'" +echo "Using bigchaindb branch '$STACK_BRANCH'" + +git clone https://github.com/${stack_repo}.git -b ${stack_branch} || true + +# Source utility functions +source ${SCRIPTS_DIR}/functions-common + +if [[ $stack_type == "local" ]]; then + mongo_version=$(echo "$mongo_version" | cut -d. -f-2) +fi + +# configure stack-config.yml +cat >$TOP_DIR/bigchaindb/pkg/configuration/vars/stack-config.yml < $CONF_DIR/hosts/all << EOF + $(hostname) ansible_connection=local +EOF + + ansible-playbook $CONF_DIR/bigchaindb-stop.yml -i $CONF_DIR/hosts/all \ + --extra-vars "operation=stop home_path=${TOP_DIR}" +else + echo "Invalid Stack Type OR Provider" + exit 1 +fi + +# Kill background processes on exit +trap exit_trap EXIT +function exit_trap { + exit $? +} +# Exit on any errors so that errors don't compound and kill if any services already started +trap err_trap ERR +function err_trap { + local r=$? + tmux kill-session bdb-dev + set +o xtrace + exit $? +} + +echo -e "Finished unstacking!!" +set -o errexit diff --git a/proposals/extend-post-txn.md b/proposals/extend-post-txn.md new file mode 100644 index 0000000..b763978 --- /dev/null +++ b/proposals/extend-post-txn.md @@ -0,0 +1,76 @@ + + +## Feature : Add query parameter "mode" to `POST` transaction +Add new query parameter `mode` to the [post transaction api](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html#post--api-v1-transactions) such that client can, +- asynchronously post the transaction +- post the transaction and return after it has been validated +- post the transaction and return after it has been committed + +## Problem Description +When posting a transaction it is broadcast asynchronously to Tendermint which enables the client to return immediately. Furthermore, the transaction status API would allow the client to get the current status for a given transaction. The above workflow seems efficient when the client doesn't need to wait until a transaction gets committed. In case a client wishes to wait until a transaction gets committed it would need to poll the transaction status api. +The Tendermint api allows to post a transaction in [three modes](https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api), + +- `/broadcast_tx_async` post transaction and return +- `/broadcast_tx_sync` post transaction and return after `checkTx` is executed +- `/broadcast_tx_commit` post transaction and return after the transaction has been committed + +### Use cases +- allow clients to post a transaction synchronously i.e. the `POST` transaction request returns after the transaction has passed `checkTx`. +- allow client to post a transaction and return after a transaction has been committed. + +## Proposed change +Add query parameter `mode` to the [`POST` transaction api](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html#post--api-v1-transactions) + +### Alternatives +N/A + +### Data model impact +N/A + +### API impact +The query parameter `mode` will be introduced to [`POST /api/v1/transaction`](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html#post--api-v1-transactions) and will have the following possible values +- `async` +- `sync` +- `commit` + +To preserve compability with the existing behavour of the API, the query parameter `mode` is optional. In this case the default value would be `async`. + + +example URI: `/api/v1/transaction?mode=async` + +### Security impact +N/A + +### Performance impact +N/A + +### End user impact +The feature itself will not impact the client drivers but it may lead to [`GET /api/v1/statuses?transaction_id={transaction_id}`](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html#get--api-v1-statuses?transaction_id=transaction_id) being deprecated in the future. + +### Deployment impact +N/A + +### Documentation impact +The documentation for [posting a transaction](https://docs.bigchaindb.com/projects/server/en/latest/http-client-server-api.html#post--api-v1-transactions) would need to describe the use of query parameter `mode`. + +### Testing impact +Following new test cases should be included +- `POST` a transaction with no `mode` i.e. `POST /api/v1/transaction` +- `POST` a transaction with `mode=sync`, `mode=async`, `mode=commit` +- `POST` a transaction with invalid `mode` + +## Implementation + +### Assignee(s) +Primary assignee(s): @kansi + +### Targeted Release +Planetmint 2.0 + +## Dependencies +N/A diff --git a/proposals/integration-test-cases.md b/proposals/integration-test-cases.md new file mode 100644 index 0000000..d35bfdb --- /dev/null +++ b/proposals/integration-test-cases.md @@ -0,0 +1,109 @@ + + +# Integration test case suggestions +This document gives an overview of possible integration test cases, provides some useful links and a specification how to write the Python `docstring` for a test case. + + +## Useful links +- testing advice by [bitcoin](https://github.com/bitcoin/bitcoin/tree/master/test/functional#general-test-writing-advice) + +- [tendermint](https://github.com/tendermint/tendermint/tree/master/test) integration tests + + +## How to structure a test scenario +The following serves as a structure to describe the tests. Each integration test should contain this description in the docstring. + +| Keyword | Description | +|-----------------------|---------------------------| +| Name | Name of the test | +| Startup State | Required base settings | +| Test Description | Steps to be executed | +| Output Specification | Expected output | + +### Startup State +The startup state specifies the system at the beginning of a test. Some questions that need to be answered: +- How many nodes will be running? +- What is the state of each node? +- Are there any faulty nodes? +- Is there any initial data setup? + - e.g. are there existing transactions? + +### Test description +Write down the steps that are executed during the tests. + +### Output Specification +This specification describes the state of the system at the end of the test. The questions in the startup state can be used. + +### Example Docstring +``` +def test_node_count(some_args): + """ + Name: Test Node Count + Startup State: None + Test Description: Start the system with 4 Nodes + Output Specification: Every node has N-1 other peers + """ + ``` + +## Scenario groups and test case suggestions +### Starting +- start n nodes, all are visible -> assert everyone has N-1 other peers +- start n nodes, one crashes, node count changes +- start nodes with different versions of bdb +### Syncing +- start n nodes, get sync, all have the same +- start n nodes, different sync, bft ok +- start n nodes, different sync, bft fails +- start n nodes, give initial blockchain and check if everyone has the correct status +- start n nodes, give initial blockchain, some faulty nodes, check status +- start n nodes, how long should sync take (timeout?) +### Crash nodes +- start n nodes, ones freezes, what is supposed to happen? +- start n nodes, one crashes, comes up, correct sync +- start n nodes, crash all, come up, correct status +### Crash components +- start n nodes, mongodb crashes +- start n nodes, tendermint crashes +- start n nodes, bigchain crashes +- start n nodes, connection crashes +- what else can crash? +- possible crash times + - on startup + - when running and nodes are synced + - on sync + - on send tx + - on new block + - on vote +### System settings +- start n nodes, have different times in nodes (timestamps) +- clock drifting (timejacking) +- start n nodes, have one key not in the keyring +### Transactions +- start n nodes, one sends tx, sync +- start n nodes, n nodes send a tx, sync +- start n nodes, one tries to double spend +- start n nodes, one sends tx, new node up, sees tx +- start n nodes, one sends divisible tx for two other nodes +### Validation +- start n nodes, check app hash +- start n nodes, check app hash, one crashes, gets up, check hash +- nodes validate tx +### Voting +- n nodes vote, bft scenarios +### Blocks +- start n nodes, one creates a block +- start n nodes, check block height, new block, check block height +- have an invalid block (new block, wrong hash?) +- have block bigger than max size +### Query +- start n nodes, let all query for the same block +- query tx +### Malicious nodes +- start n nodes, one manipulates the blockchain +### Events +- start n nodes, let one check for event stream of another diff --git a/proposals/integration-testing.md b/proposals/integration-testing.md new file mode 100644 index 0000000..d860055 --- /dev/null +++ b/proposals/integration-testing.md @@ -0,0 +1,117 @@ + + +# Integration testing tools for Planetmint + +## Problem Description +We need a proper way to describe, code, and run integration tests. While we have unit tests to verify the correct behavior of functions and methods, we need a way to easily test a network of Planetmint nodes. + +As an example, we want to make sure that if a valid transaction is pushed to the network, after some time that transaction is stored by more than 2/3 of the nodes; an invalid transaction (malformed, double spent, etc.) pushed to the network must be rejected. This must be true also in different scenarios such as not all nodes are up, or there is latency between the nodes. Note that some of those problems have already been addressed by the Tendermint team. + +### Use cases +- Verify we don't break the contract with the user. +- Verify the behavior of the system in case of failure of its services (MongoDB fails, Tendermint fails, Planetmint fails) +- Verify that some properties of the system are implemented correctly. + +## Feature: add a client to manage a Planetmint network +We define now the characteristics of a new module: the Network Driver, `ND` henceforth. As any other project, it needs a catchy name, but I don't have one in my head now. + +The goal of `ND` is to give a simple framework to implement test scenarios that require: +- creating a network +- turn the nodes in the network up or down +- turn the services of a node up or down + +There are other useful features we can take into consideration, like: +- creating and pushing transactions to one or more nodes in the network +- check the existence of a transaction in one or more nodes in the network + +*Transaction specific* features can be added to `ND` later. We can just use the Planetmint Python driver for now, or raw `HTTP` requests. + +[Bitcoin functional tests](https://github.com/bitcoin/bitcoin/tree/v0.15.0/test/functional) have as similar approach (thanks @codegeschrei for the link). + +## Proposed Change +First of all, a new directory will be created, namely `integration-tests`, in the root of the project. This directory will contain the code for `ND`, and will be home for the new integration tests. + +Integration tests require not only one node, but a whole network to be up and running. A `docker-compose` configuration file has been created for that purpose, and the `ND` module should leverage it. + +The `ND` module implements two main functions: +- network setup, using `docker-compose` +- basic transaction management + +In the next sections we will use `ND` to refer to the Python class, and `network` to refer to an instance of it. For now, `network` will most likely be a singleton, since it will control `docker` in the current host. This **will not** be the final name of the module. + +### Usage example +The following code is just a suggestion on how the new module shuold be used. It may contain syntax errors or other kind of errors. + +```python +def test_valid_transaction_is_available_in_all_nodes(network): + alice, bob = crypto.generate_key_pair(), crypto.generate_key_pair() + tx_alice_to_bob = Transaction.create([alice.public_key], [([bob.public_key], 1)])\ + .sign([alice.private_key]) + tx = tx.sign([user_priv]) + network.start(4) + network.node[0].stop() + requests.post(network.node[1].url + 'transactions', data=json.dumps(tx.to_dict())) + network.node[0].start() + + time.sleep(1.5) + + for node in network: + assert requests.get(node.url + 'transactions/' + tx.id) == tx + + network.stop() +``` + +To facilitate testing and ease the integration with the test framework `pytest`, fixtures will be provided. + +### Alternatives +One can argue that the tests should reside in the `tests` directory. This makes sense, but the new test suite has different requirements in terms of fixtures. + +[Behavioral Driven Development](https://en.wikipedia.org/wiki/Behavior-driven_development) can be an inspiration for this kind of testing, since it allows to define what is the behavior of the **whole system** given some constraints. Moreover, it's easy to understand since plain English plays a key role. [`pytest-bdd`](https://pypi.python.org/pypi/pytest-bdd) is a plugin for `pytest` that adds fixtures to handle this kind of testing. + +### Data model impact +N/A + +### API impact +N/A + +### Security impact +N/A + +### Performance impact +N/A + +### End user impact +N/A + +### Deployment impact +The new test suite should be deployed in the CI system. + +### Documentation impact +The new test suite should be referenced in the documentation. + + +### Testing impact +N/A + + +## Implementation + +### Assignee(s) +Primary assignee(s): @vrde + + +### Targeted Release +Planetmint 2.0 + + +## Dependencies +N/A + +## Reference(s) +* [Fixture for acceptance tests](https://github.com/bigchaindb/bigchaindb/pull/1384) +* [Test network Deployment: 4 nodes](https://github.com/bigchaindb/bigchaindb/issues/1922) diff --git a/proposals/migrate-cli.md b/proposals/migrate-cli.md new file mode 100644 index 0000000..1c0c52e --- /dev/null +++ b/proposals/migrate-cli.md @@ -0,0 +1,169 @@ + + +# Migrate Bigchaindb cli for Tendermint + +## Problem Description +With Tendermint integration some of the cli sub-commands have been rendered obsolete. It would be only appropriate to remove those sub-commands. + +### Use cases +- Avoid confusing the user by not displaying irrelevant sub-commands. + + +## Proposed Change +Following sub-commands should be updated/removed: + +- `bigchaindb --help`: list the relevant sub-commands for `localmongodb` backend. +`mongodb` and `rethinkdb` will be deprecated. +In case the backend is not configured then the default backend `localmongodb` should be assumed. + +Following sub-commands should be deprecated for `localmongodb` backend. + +- `bigchaindb export-my-pubkey` + - A Planetmint node still has a public key but that is not Planetmint concern. It is handled by Tendermint. +- `bigchaindb set-shards` + - This was only required for `rethinkdb`. +- `bigchaindb set-replicas` + - This was only required for `rethinkdb`. +- `bigchaindb add-replicas` + - This was only required for `mongodb` backend to add nodes to the MongoDB Replica Set, which is not required anymore, + because we are using standalone MongoDB instances i.e. `localmongodb`. +- `bigchaindb remove-replicas` + - This was only required for backend to remove nodes from the MongoDB Replica Set, which is not required anymore. + +### Usage example +**bigchaindb** + +``` +$ bigchaindb --help +usage: bigchaindb [-h] [-c CONFIG] [-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}] + [-y] [-v] + {configure,show-config,init,drop,start} + ... + +Control your Planetmint node. + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + Specify the location of the configuration file (use + "-" for stdout) + -l {DEBUG,INFO,WARNING,ERROR,CRITICAL}, --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL} + Log level + -y, --yes, --yes-please + Assume "yes" as answer to all prompts and run non- + interactively + -v, --version show program's version number and exit + +Commands: + {configure,show-config,export-my-pubkey,init,drop,start,set-shards,set-replicas,add-replicas,remove-replicas} + configure Prepare the config file + show-config Show the current configuration + init Init the database + drop Drop the database + start Start Planetmint +``` + +**bigchaindb configure** + +``` +$ bigchaindb configure --help +usage: bigchaindb configure [-h] {localmongodb} + +positional arguments: + {localmongodb} The backend to use. It can be only be `localmongodb`. + +optional arguments: + -h, --help show this help message and exit +``` + +**bigchaindb show-config** + +``` +$ bigchaindb show-config --help +usage: bigchaindb show-config [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +**bigchaindb init** + +``` +$ bigchaindb init --help +usage: bigchaindb init [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +**bigchaindb drop** + +``` +$ bigchaindb drop --help +usage: bigchaindb drop [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +**bigchaindb start** + +``` +$ bigchaindb start --help +usage: bigchaindb start [-h] +optional arguments: + -h, --help show this help message and exit +``` + +### Data model impact +N/A + +### API impact +N/A + +### Security impact +N/A + +### Performance impact +N/A + +### End user impact +N/A + +### Deployment impact +N/A + +### Documentation impact +Document the commands and sub-commands along with usage. + + +### Testing impact +Following test cases should be added +- Set a backend other than `localmongodb` and see of it results in a valid unsupported + result. +- Set `localmongodb` as backend and execute `bigchaindb --help` and validate that only the above + mentioned sub-commands are displayed. + + +## Implementation + +### Assignee(s) +Primary assignee(s): @muawiakh + +Secondary assignee(s): @kansi, @ttmc + +### Targeted Release +Planetmint 2.0 + + +## Dependencies +N/A + + +## Reference(s) +* [Bigchaindb CLI](https://docs.bigchaindb.com/projects/server/en/latest/server-reference/bigchaindb-cli.html) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..625c3e0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = tests/ +norecursedirs = .* *.egg *.egg-info env* devenv* docs +addopts = -m "not abci" +looponfailroots = bigchaindb tests diff --git a/run-acceptance-test.sh b/run-acceptance-test.sh new file mode 100755 index 0000000..5daa44e --- /dev/null +++ b/run-acceptance-test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +# Set up a Planetmint node and return only when we are able to connect to both +# the Planetmint container *and* the Tendermint container. +setup () { + docker-compose up -d bigchaindb + + # Try to connect to the containers for maximum three times, and wait + # one second between tries. + for i in $(seq 3); do + if $(docker-compose run --rm curl-client); then + break + else + sleep 1 + fi + done +} + +run_test () { + docker-compose run --rm python-acceptance pytest /src +} + +teardown () { + docker-compose down +} + +setup +run_test +exitcode=$? +teardown + +exit $exitcode diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..cdec23f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,9 @@ +[aliases] +test=pytest + +[coverage:run] +source = . +omit = *test* + +[flake8] +max_line_length = 119 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..946b185 --- /dev/null +++ b/setup.py @@ -0,0 +1,149 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +""" +Planetmint: The Blockchain Database + +For full docs visit https://docs.bigchaindb.com + +""" + +import sys +from setuptools import setup, find_packages + +if sys.version_info < (3, 6): + sys.exit('Please use Python version 3.6 or higher.') + +# get the version +version = {} +with open('bigchaindb/version.py') as fp: + exec(fp.read(), version) + +def check_setuptools_features(): + """Check if setuptools is up to date.""" + import pkg_resources + try: + list(pkg_resources.parse_requirements('foo~=1.0')) + except ValueError: + sys.exit('Your Python distribution comes with an incompatible version ' + 'of `setuptools`. Please run:\n' + ' $ pip3 install --upgrade setuptools\n' + 'and then run this command again') + + +check_setuptools_features() + +dev_require = [ + 'ipdb', + 'ipython', + 'watchdog', + 'logging_tree', + 'pre-commit' +] + +docs_require = [ + 'Sphinx~=1.0', + 'recommonmark>=0.4.0', + 'sphinx-rtd-theme>=0.1.9', + 'sphinxcontrib-httpdomain>=1.5.0', + 'sphinxcontrib-napoleon>=0.4.4', + 'aafigure>=0.6', + 'wget' +] + +tests_require = [ + 'coverage', + 'pep8', + 'flake8', + 'flake8-quotes==0.8.1', + 'hypothesis>=5.3.0', + # Removed pylint because its GPL license isn't Apache2-compatible + 'pytest>=3.0.0', + 'pytest-cov==2.8.1', + 'pytest-mock', + 'pytest-xdist', + 'pytest-flask', + 'pytest-aiohttp', + 'pytest-asyncio', + 'tox', +] + docs_require + +install_requires = [ + 'aiohttp==3.7.4', + 'bigchaindb-abci==1.0.7', + 'cryptoconditions==0.8.1', + 'flask-cors==3.0.10', + 'flask-restful==0.3.9', + 'flask==2.0.1', + 'gunicorn==20.1.0', + 'jsonschema==3.2.0', + 'logstats==0.3.0', + 'packaging>=20.9', + # TODO Consider not installing the db drivers, or putting them in extras. + 'pymongo==3.11.4', + 'python-rapidjson==1.0', + 'pyyaml==5.4.1', + 'requests==2.25.1', + 'setproctitle==1.2.2', +] + +if sys.version_info < (3, 6): + install_requires.append('pysha3~=1.0.2') + +setup( + name='Planetmint', + version=version['__version__'], + description='Planetmint: The Blockchain Database', + long_description=( + "Planetmint allows developers and enterprises to deploy blockchain " + "proof-of-concepts, platforms and applications with a blockchain " + "database. Planetmint supports a wide range of industries and use cases " + "from identity and intellectual property to supply chains, energy, IoT " + "and financial ecosystems. With high throughput, low latency, powerful " + "query functionality, decentralized control, immutable data storage and " + "built-in asset support, Planetmint is like a database with blockchain " + "characteristics." + ), + url='https://github.com/Planetmint/bigchaindb/', + author='Planetmint Contributors', + author_email='contact@ipdb.global', + license='Apache Software License 2.0', + zip_safe=False, + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Topic :: Database', + 'Topic :: Database :: Database Engines/Servers', + 'Topic :: Software Development', + 'Natural Language :: English', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX :: Linux', + ], + + packages=find_packages(exclude=['tests*']), + + scripts=['pkg/scripts/bigchaindb-monit-config'], + + entry_points={ + 'console_scripts': [ + 'bigchaindb=bigchaindb.commands.bigchaindb:main' + ], + }, + install_requires=install_requires, + setup_requires=['pytest-runner'], + tests_require=tests_require, + extras_require={ + 'test': tests_require, + 'dev': dev_require + tests_require + docs_require, + 'docs': docs_require, + }, + package_data={'bigchaindb.common.schema': ['*.yaml']}, +) diff --git a/snap/README.md b/snap/README.md new file mode 100644 index 0000000..b5c6ad3 --- /dev/null +++ b/snap/README.md @@ -0,0 +1,19 @@ + + +This is the packaging metadata for the Planetmint snap. + +Snaps and the snap store allows for the secure installation of apps that work +in most Linux distributions. For more information, go to https://snapcraft.io/ + +To build and install this snap in Ubuntu 16.04: + + $ sudo apt install git snapcraft + $ git clone https://github.com/bigchaindb/bigchaindb + $ cd bigchaindb + $ snapcraft + $ sudo snap install *.snap --dangerous --devmode diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 0000000..fc1ec55 --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,29 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +name: bigchaindb +version: git +summary: The blockchain database +description: | + With high throughput, low latency, powerful query functionality, + decentralized control, immutable data storage and built-in asset + support, Planetmint is like a database with blockchain + characteristics. + +# grade must be 'stable' to release into candidate/stable channels +grade: devel +# strict confinement requires https://github.com/snapcore/snapd/pull/2749 +confinement: devmode + +apps: + bigchaindb: + command: bigchaindb + plugs: [network, network-bind] + +parts: + bigchaindb: + source: . + plugin: python + build-packages: [g++, libffi-dev] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..959bbf1 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,17 @@ + + +# Planetmint Server Unit Tests + +Most of the tests in the `tests/` folder are unit tests. For info about how to write and run tests, see [the docs about contributing to Planetmint](http://docs.bigchaindb.com/projects/contributing/en/latest/index.html), especially: + +- [Write Code - Remember to Write Tests](http://docs.bigchaindb.com/projects/contributing/en/latest/dev-setup-coding-and-contribution-process/write-code.html#remember-to-write-tests) +- [Notes on Running a Local Dev Node with Docker Compose](http://docs.bigchaindb.com/projects/contributing/en/latest/dev-setup-coding-and-contribution-process/run-node-with-docker-compose.html), especially `make test` +- [ +Notes on Running a Local Dev Node as Processes (and Running All Tests)](http://docs.bigchaindb.com/projects/contributing/en/latest/dev-setup-coding-and-contribution-process/run-node-as-processes.html) + +Note: There are acceptance tests in the `acceptance/` folder (at the same level in the hierarchy as the `tests/` folder). diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/assets/__init__.py b/tests/assets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/assets/test_digital_assets.py b/tests/assets/test_digital_assets.py new file mode 100644 index 0000000..e24a7b9 --- /dev/null +++ b/tests/assets/test_digital_assets.py @@ -0,0 +1,73 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest +import random + + +def test_asset_transfer(b, signed_create_tx, user_pk, user_sk): + from bigchaindb.models import Transaction + + tx_transfer = Transaction.transfer(signed_create_tx.to_inputs(), [([user_pk], 1)], + signed_create_tx.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([signed_create_tx]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert tx_transfer_signed.asset['id'] == signed_create_tx.id + + +def test_validate_transfer_asset_id_mismatch(b, signed_create_tx, user_pk, user_sk): + from bigchaindb.common.exceptions import AssetIdMismatch + from bigchaindb.models import Transaction + + tx_transfer = Transaction.transfer(signed_create_tx.to_inputs(), [([user_pk], 1)], + signed_create_tx.id) + tx_transfer.asset['id'] = 'a' * 64 + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([signed_create_tx]) + + with pytest.raises(AssetIdMismatch): + tx_transfer_signed.validate(b) + + +def test_get_asset_id_create_transaction(alice, user_pk): + from bigchaindb.models import Transaction + tx_create = Transaction.create([alice.public_key], [([user_pk], 1)]) + assert Transaction.get_asset_id(tx_create) == tx_create.id + + +def test_get_asset_id_transfer_transaction(b, signed_create_tx, user_pk): + from bigchaindb.models import Transaction + + tx_transfer = Transaction.transfer(signed_create_tx.to_inputs(), [([user_pk], 1)], + signed_create_tx.id) + asset_id = Transaction.get_asset_id(tx_transfer) + assert asset_id == tx_transfer.asset['id'] + + +def test_asset_id_mismatch(alice, user_pk): + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import AssetIdMismatch + + tx1 = Transaction.create([alice.public_key], [([user_pk], 1)], + metadata={'msg': random.random()}) + tx1.sign([alice.private_key]) + tx2 = Transaction.create([alice.public_key], [([user_pk], 1)], + metadata={'msg': random.random()}) + tx2.sign([alice.private_key]) + + with pytest.raises(AssetIdMismatch): + Transaction.get_asset_id([tx1, tx2]) + + +def test_create_valid_divisible_asset(b, user_pk, user_sk): + from bigchaindb.models import Transaction + + tx = Transaction.create([user_pk], [([user_pk], 2)]) + tx_signed = tx.sign([user_sk]) + assert tx_signed.validate(b) == tx_signed diff --git a/tests/assets/test_divisible_assets.py b/tests/assets/test_divisible_assets.py new file mode 100644 index 0000000..cd3cb13 --- /dev/null +++ b/tests/assets/test_divisible_assets.py @@ -0,0 +1,593 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest +import random + +from bigchaindb.common.exceptions import DoubleSpend + + +# CREATE divisible asset +# Single input +# Single owners_before +# Single output +# Single owners_after +def test_single_in_single_own_single_out_single_own_create(alice, user_pk, b): + from bigchaindb.models import Transaction + + tx = Transaction.create([alice.public_key], [([user_pk], 100)], asset={'name': random.random()}) + tx_signed = tx.sign([alice.private_key]) + + assert tx_signed.validate(b) == tx_signed + assert len(tx_signed.outputs) == 1 + assert tx_signed.outputs[0].amount == 100 + assert len(tx_signed.inputs) == 1 + + +# CREATE divisible asset +# Single input +# Single owners_before +# Multiple outputs +# Single owners_after per output +def test_single_in_single_own_multiple_out_single_own_create(alice, user_pk, b): + from bigchaindb.models import Transaction + + tx = Transaction.create([alice.public_key], [([user_pk], 50), ([user_pk], 50)], + asset={'name': random.random()}) + tx_signed = tx.sign([alice.private_key]) + + assert tx_signed.validate(b) == tx_signed + assert len(tx_signed.outputs) == 2 + assert tx_signed.outputs[0].amount == 50 + assert tx_signed.outputs[1].amount == 50 + assert len(tx_signed.inputs) == 1 + + +# CREATE divisible asset +# Single input +# Single owners_before +# Single output +# Multiple owners_after +def test_single_in_single_own_single_out_multiple_own_create(alice, user_pk, b): + from bigchaindb.models import Transaction + + tx = Transaction.create([alice.public_key], [([user_pk, user_pk], 100)], asset={'name': random.random()}) + tx_signed = tx.sign([alice.private_key]) + + assert tx_signed.validate(b) == tx_signed + assert len(tx_signed.outputs) == 1 + assert tx_signed.outputs[0].amount == 100 + + output = tx_signed.outputs[0].to_dict() + assert 'subconditions' in output['condition']['details'] + assert len(output['condition']['details']['subconditions']) == 2 + + assert len(tx_signed.inputs) == 1 + + +# CREATE divisible asset +# Single input +# Single owners_before +# Multiple outputs +# Mix: one output with a single owners_after, one output with multiple +# owners_after +def test_single_in_single_own_multiple_out_mix_own_create(alice, user_pk, b): + from bigchaindb.models import Transaction + + tx = Transaction.create([alice.public_key], [([user_pk], 50), ([user_pk, user_pk], 50)], + asset={'name': random.random()}) + tx_signed = tx.sign([alice.private_key]) + + assert tx_signed.validate(b) == tx_signed + assert len(tx_signed.outputs) == 2 + assert tx_signed.outputs[0].amount == 50 + assert tx_signed.outputs[1].amount == 50 + + output_cid1 = tx_signed.outputs[1].to_dict() + assert 'subconditions' in output_cid1['condition']['details'] + assert len(output_cid1['condition']['details']['subconditions']) == 2 + + assert len(tx_signed.inputs) == 1 + + +# CREATE divisible asset +# Single input +# Multiple owners_before +# Output combinations already tested above +def test_single_in_multiple_own_single_out_single_own_create(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details + + tx = Transaction.create([alice.public_key, user_pk], [([user_pk], 100)], asset={'name': random.random()}) + tx_signed = tx.sign([alice.private_key, user_sk]) + assert tx_signed.validate(b) == tx_signed + assert len(tx_signed.outputs) == 1 + assert tx_signed.outputs[0].amount == 100 + assert len(tx_signed.inputs) == 1 + + ffill = _fulfillment_to_details(tx_signed.inputs[0].fulfillment) + assert 'subconditions' in ffill + assert len(ffill['subconditions']) == 2 + + +# TRANSFER divisible asset +# Single input +# Single owners_before +# Single output +# Single owners_after +def test_single_in_single_own_single_out_single_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 100)], asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 100 + assert len(tx_transfer_signed.inputs) == 1 + + +# TRANSFER divisible asset +# Single input +# Single owners_before +# Multiple output +# Single owners_after +def test_single_in_single_own_multiple_out_single_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 100)], asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), + [([alice.public_key], 50), ([alice.public_key], 50)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 2 + assert tx_transfer_signed.outputs[0].amount == 50 + assert tx_transfer_signed.outputs[1].amount == 50 + assert len(tx_transfer_signed.inputs) == 1 + + +# TRANSFER divisible asset +# Single input +# Single owners_before +# Single output +# Multiple owners_after +def test_single_in_single_own_single_out_multiple_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 100)], asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), + [([alice.public_key, alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 100 + + condition = tx_transfer_signed.outputs[0].to_dict() + assert 'subconditions' in condition['condition']['details'] + assert len(condition['condition']['details']['subconditions']) == 2 + + assert len(tx_transfer_signed.inputs) == 1 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Single input +# Single owners_before +# Multiple outputs +# Mix: one output with a single owners_after, one output with multiple +# owners_after +def test_single_in_single_own_multiple_out_mix_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 100)], asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), + [([alice.public_key], 50), ([alice.public_key, alice.public_key], 50)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 2 + assert tx_transfer_signed.outputs[0].amount == 50 + assert tx_transfer_signed.outputs[1].amount == 50 + + output_cid1 = tx_transfer_signed.outputs[1].to_dict() + assert 'subconditions' in output_cid1['condition']['details'] + assert len(output_cid1['condition']['details']['subconditions']) == 2 + + assert len(tx_transfer_signed.inputs) == 1 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Single input +# Multiple owners_before +# Single output +# Single owners_after +def test_single_in_multiple_own_single_out_single_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([alice.public_key, user_pk], 100)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([alice.private_key, user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 100 + assert len(tx_transfer_signed.inputs) == 1 + + ffill = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + assert 'subconditions' in ffill + assert len(ffill['subconditions']) == 2 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Multiple inputs +# Single owners_before per input +# Single output +# Single owners_after +def test_multiple_in_single_own_single_out_single_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 50), ([user_pk], 50)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 100 + assert len(tx_transfer_signed.inputs) == 2 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Multiple inputs +# Multiple owners_before per input +# Single output +# Single owners_after +def test_multiple_in_multiple_own_single_out_single_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk, alice.public_key], 50), + ([user_pk, alice.public_key], 50)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([alice.private_key, user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 100 + assert len(tx_transfer_signed.inputs) == 2 + + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid0['subconditions']) == 2 + assert len(ffill_fid1['subconditions']) == 2 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Multiple inputs +# Mix: one input with a single owners_before, one input with multiple +# owners_before +# Single output +# Single owners_after +def test_muiltiple_in_mix_own_multiple_out_single_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 50), ([user_pk, alice.public_key], 50)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([alice.private_key, user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 100 + assert len(tx_transfer_signed.inputs) == 2 + + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' not in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid1['subconditions']) == 2 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Multiple inputs +# Mix: one input with a single owners_before, one input with multiple +# owners_before +# Multiple outputs +# Mix: one output with a single owners_after, one output with multiple +# owners_after +def test_muiltiple_in_mix_own_multiple_out_mix_own_transfer(alice, b, user_pk, + user_sk): + from bigchaindb.models import Transaction + from bigchaindb.common.transaction import _fulfillment_to_details + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 50), ([user_pk, alice.public_key], 50)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), + [([alice.public_key], 50), ([alice.public_key, user_pk], 50)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([alice.private_key, user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 2 + assert tx_transfer_signed.outputs[0].amount == 50 + assert tx_transfer_signed.outputs[1].amount == 50 + assert len(tx_transfer_signed.inputs) == 2 + + cond_cid0 = tx_transfer_signed.outputs[0].to_dict() + cond_cid1 = tx_transfer_signed.outputs[1].to_dict() + assert 'subconditions' not in cond_cid0['condition']['details'] + assert 'subconditions' in cond_cid1['condition']['details'] + assert len(cond_cid1['condition']['details']['subconditions']) == 2 + + ffill_fid0 = _fulfillment_to_details(tx_transfer_signed.inputs[0].fulfillment) + ffill_fid1 = _fulfillment_to_details(tx_transfer_signed.inputs[1].fulfillment) + assert 'subconditions' not in ffill_fid0 + assert 'subconditions' in ffill_fid1 + assert len(ffill_fid1['subconditions']) == 2 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +# TRANSFER divisible asset +# Multiple inputs from different transactions +# Single owners_before +# Single output +# Single owners_after +def test_multiple_in_different_transactions(alice, b, user_pk, user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset + # `b` creates a divisible asset and assigns 50 shares to `b` and + # 50 shares to `user_pk` + tx_create = Transaction.create([alice.public_key], [([user_pk], 50), ([alice.public_key], 50)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER divisible asset + # `b` transfers its 50 shares to `user_pk` + # after this transaction `user_pk` will have a total of 100 shares + # split across two different transactions + tx_transfer1 = Transaction.transfer(tx_create.to_inputs([1]), + [([user_pk], 50)], + asset_id=tx_create.id) + tx_transfer1_signed = tx_transfer1.sign([alice.private_key]) + + # TRANSFER + # `user_pk` combines two different transaction with 50 shares each and + # transfers a total of 100 shares back to `b` + tx_transfer2 = Transaction.transfer(tx_create.to_inputs([0]) + + tx_transfer1.to_inputs([0]), + [([alice.private_key], 100)], + asset_id=tx_create.id) + tx_transfer2_signed = tx_transfer2.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed, tx_transfer1_signed]) + + assert tx_transfer2_signed.validate(b) == tx_transfer2_signed + assert len(tx_transfer2_signed.outputs) == 1 + assert tx_transfer2_signed.outputs[0].amount == 100 + assert len(tx_transfer2_signed.inputs) == 2 + + fid0_input = tx_transfer2_signed.inputs[0].fulfills.txid + fid1_input = tx_transfer2_signed.inputs[1].fulfills.txid + assert fid0_input == tx_create.id + assert fid1_input == tx_transfer1.id + + +# In a TRANSFER transaction of a divisible asset the amount being spent in the +# inputs needs to match the amount being sent in the outputs. +# In other words `amount_in_inputs - amount_in_outputs == 0` +def test_amount_error_transfer(alice, b, user_pk, user_sk): + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import AmountError + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 100)], asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + b.store_bulk_transactions([tx_create_signed]) + + # TRANSFER + # output amount less than input amount + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 50)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + with pytest.raises(AmountError): + tx_transfer_signed.validate(b) + + # TRANSFER + # output amount greater than input amount + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 101)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + with pytest.raises(AmountError): + tx_transfer_signed.validate(b) + + +def test_threshold_same_public_key(alice, b, user_pk, user_sk): + # If we try to fulfill a threshold condition where each subcondition has + # the same key get_subcondition_from_vk will always return the first + # subcondition. This means that only the 1st subfulfillment will be + # generated + # Creating threshold conditions with the same key does not make sense but + # that does not mean that the code shouldn't work. + + from bigchaindb.models import Transaction + + # CREATE divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk, user_pk], 100)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # TRANSFER + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 100)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk, user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +def test_sum_amount(alice, b, user_pk, user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset with 3 outputs with amount 1 + tx_create = Transaction.create([alice.public_key], [([user_pk], 1), ([user_pk], 1), ([user_pk], 1)], + asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # create a transfer transaction with one output and check if the amount + # is 3 + tx_transfer = Transaction.transfer(tx_create.to_inputs(), [([alice.public_key], 3)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 1 + assert tx_transfer_signed.outputs[0].amount == 3 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +def test_divide(alice, b, user_pk, user_sk): + from bigchaindb.models import Transaction + + # CREATE divisible asset with 1 output with amount 3 + tx_create = Transaction.create([alice.public_key], [([user_pk], 3)], asset={'name': random.random()}) + tx_create_signed = tx_create.sign([alice.private_key]) + + # create a transfer transaction with 3 outputs and check if the amount + # of each output is 1 + tx_transfer = Transaction.transfer(tx_create.to_inputs(), + [([alice.public_key], 1), ([alice.public_key], 1), ([alice.public_key], 1)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + + b.store_bulk_transactions([tx_create_signed]) + + assert tx_transfer_signed.validate(b) == tx_transfer_signed + assert len(tx_transfer_signed.outputs) == 3 + for output in tx_transfer_signed.outputs: + assert output.amount == 1 + + b.store_bulk_transactions([tx_transfer_signed]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) diff --git a/tests/backend/__init__.py b/tests/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/backend/localmongodb/__init__.py b/tests/backend/localmongodb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/backend/localmongodb/conftest.py b/tests/backend/localmongodb/conftest.py new file mode 100644 index 0000000..7c1f2d6 --- /dev/null +++ b/tests/backend/localmongodb/conftest.py @@ -0,0 +1,17 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from pymongo import MongoClient +from pytest import fixture + + +@fixture +def mongo_client(db_context): + return MongoClient(host=db_context.host, port=db_context.port) + + +@fixture +def utxo_collection(db_context, mongo_client): + return mongo_client[db_context.name].utxos diff --git a/tests/backend/localmongodb/test_connection.py b/tests/backend/localmongodb/test_connection.py new file mode 100644 index 0000000..bd4294a --- /dev/null +++ b/tests/backend/localmongodb/test_connection.py @@ -0,0 +1,111 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from unittest import mock + +import pytest +import pymongo +from pymongo import MongoClient + + +pytestmark = pytest.mark.bdb + + +@pytest.fixture +def mock_cmd_line_opts(): + return {'argv': ['mongod', '--dbpath=/data'], + 'ok': 1.0, + 'parsed': {'replication': {'replSet': None}, + 'storage': {'dbPath': '/data'}}} + + +@pytest.fixture +def mock_config_opts(): + return {'argv': ['mongod', '--dbpath=/data'], + 'ok': 1.0, + 'parsed': {'replication': {'replSetName': None}, + 'storage': {'dbPath': '/data'}}} + + +@pytest.fixture +def mongodb_connection(): + import bigchaindb + return MongoClient(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port']) + + +def test_get_connection_returns_the_correct_instance(db_host, db_port): + from bigchaindb.backend import connect + from bigchaindb.backend.connection import Connection + from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection + + config = { + 'backend': 'localmongodb', + 'host': db_host, + 'port': db_port, + 'name': 'test', + 'replicaset': None, + } + + conn = connect(**config) + assert isinstance(conn, Connection) + assert isinstance(conn, LocalMongoDBConnection) + assert conn.conn._topology_settings.replica_set_name == config['replicaset'] + + +@mock.patch('pymongo.MongoClient.__init__') +def test_connection_error(mock_client): + from bigchaindb.backend import connect + from bigchaindb.backend.exceptions import ConnectionError + + # force the driver to throw ConnectionFailure + # the mock on time.sleep is to prevent the actual sleep when running + # the tests + mock_client.side_effect = pymongo.errors.ConnectionFailure() + + with pytest.raises(ConnectionError): + conn = connect() + conn.db + + assert mock_client.call_count == 3 + + +def test_connection_run_errors(): + from bigchaindb.backend import connect + from bigchaindb.backend.exceptions import (DuplicateKeyError, + OperationError, + ConnectionError) + + conn = connect() + + query = mock.Mock() + query.run.side_effect = pymongo.errors.AutoReconnect('foo') + with pytest.raises(ConnectionError): + conn.run(query) + assert query.run.call_count == 2 + + query = mock.Mock() + query.run.side_effect = pymongo.errors.DuplicateKeyError('foo') + with pytest.raises(DuplicateKeyError): + conn.run(query) + assert query.run.call_count == 1 + + query = mock.Mock() + query.run.side_effect = pymongo.errors.OperationFailure('foo') + with pytest.raises(OperationError): + conn.run(query) + assert query.run.call_count == 1 + + +@mock.patch('pymongo.database.Database.authenticate') +def test_connection_with_credentials(mock_authenticate): + import bigchaindb + from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection + conn = LocalMongoDBConnection(host=bigchaindb.config['database']['host'], + port=bigchaindb.config['database']['port'], + login='theplague', + password='secret') + conn.connect() + assert mock_authenticate.call_count == 1 diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py new file mode 100644 index 0000000..ec43e1e --- /dev/null +++ b/tests/backend/localmongodb/test_queries.py @@ -0,0 +1,484 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from copy import deepcopy + +import pytest +import pymongo + +from bigchaindb.backend import connect, query + + +pytestmark = pytest.mark.bdb + + +def test_get_txids_filtered(signed_create_tx, signed_transfer_tx): + from bigchaindb.backend import connect, query + from bigchaindb.models import Transaction + conn = connect() + + # create and insert two blocks, one for the create and one for the + # transfer transaction + conn.db.transactions.insert_one(signed_create_tx.to_dict()) + conn.db.transactions.insert_one(signed_transfer_tx.to_dict()) + + asset_id = Transaction.get_asset_id([signed_create_tx, signed_transfer_tx]) + + # Test get by just asset id + txids = set(query.get_txids_filtered(conn, asset_id)) + assert txids == {signed_create_tx.id, signed_transfer_tx.id} + + # Test get by asset and CREATE + txids = set(query.get_txids_filtered(conn, asset_id, Transaction.CREATE)) + assert txids == {signed_create_tx.id} + + # Test get by asset and TRANSFER + txids = set(query.get_txids_filtered(conn, asset_id, Transaction.TRANSFER)) + assert txids == {signed_transfer_tx.id} + + +def test_write_assets(): + from bigchaindb.backend import connect, query + conn = connect() + + assets = [ + {'id': 1, 'data': '1'}, + {'id': 2, 'data': '2'}, + {'id': 3, 'data': '3'}, + # Duplicated id. Should not be written to the database + {'id': 1, 'data': '1'}, + ] + + # write the assets + for asset in assets: + query.store_asset(conn, deepcopy(asset)) + + # check that 3 assets were written to the database + cursor = conn.db.assets.find({}, projection={'_id': False})\ + .sort('id', pymongo.ASCENDING) + + assert cursor.collection.count_documents({}) == 3 + assert list(cursor) == assets[:-1] + + +def test_get_assets(): + from bigchaindb.backend import connect, query + conn = connect() + + assets = [ + {'id': 1, 'data': '1'}, + {'id': 2, 'data': '2'}, + {'id': 3, 'data': '3'}, + ] + + conn.db.assets.insert_many(deepcopy(assets), ordered=False) + + for asset in assets: + assert query.get_asset(conn, asset['id']) + + +@pytest.mark.parametrize('table', ['assets', 'metadata']) +def test_text_search(table): + from bigchaindb.backend import connect, query + conn = connect() + + # Example data and tests cases taken from the mongodb documentation + # https://docs.mongodb.com/manual/reference/operator/query/text/ + objects = [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + {'id': 3, 'subject': 'Baking a cake', 'author': 'abc', 'views': 90}, + {'id': 4, 'subject': 'baking', 'author': 'xyz', 'views': 100}, + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + {'id': 6, 'subject': 'Сырники', 'author': 'jkl', 'views': 80}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, + {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} + ] + + # insert the assets + conn.db[table].insert_many(deepcopy(objects), ordered=False) + + # test search single word + assert list(query.text_search(conn, 'coffee', table=table)) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, + ] + + # match any of the search terms + assert list(query.text_search(conn, 'bake coffee cake', table=table)) == [ + {'author': 'abc', 'id': 3, 'subject': 'Baking a cake', 'views': 90}, + {'author': 'xyz', 'id': 1, 'subject': 'coffee', 'views': 50}, + {'author': 'xyz', 'id': 4, 'subject': 'baking', 'views': 100}, + {'author': 'efg', 'id': 2, 'subject': 'Coffee Shopping', 'views': 5}, + {'author': 'efg', 'id': 7, 'subject': 'coffee and cream', 'views': 10} + ] + + # search for a phrase + assert list(query.text_search(conn, '\"coffee shop\"', table=table)) == [ + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + ] + + # exclude documents that contain a term + assert list(query.text_search(conn, 'coffee -shop', table=table)) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, + ] + + # search different language + assert list(query.text_search(conn, 'leche', language='es', table=table)) == [ + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} + ] + + # case and diacritic insensitive search + assert list(query.text_search(conn, 'сы́рники CAFÉS', table=table)) == [ + {'id': 6, 'subject': 'Сырники', 'author': 'jkl', 'views': 80}, + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} + ] + + # case sensitive search + assert list(query.text_search(conn, 'Coffee', case_sensitive=True, table=table)) == [ + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + ] + + # diacritic sensitive search + assert list(query.text_search(conn, 'CAFÉ', diacritic_sensitive=True, table=table)) == [ + {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, + ] + + # return text score + assert list(query.text_search(conn, 'coffee', text_score=True, table=table)) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50, 'score': 1.0}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5, 'score': 0.75}, + {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10, 'score': 0.75}, + ] + + # limit search result + assert list(query.text_search(conn, 'coffee', limit=2, table=table)) == [ + {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, + {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, + ] + + +def test_write_metadata(): + from bigchaindb.backend import connect, query + conn = connect() + + metadata = [ + {'id': 1, 'data': '1'}, + {'id': 2, 'data': '2'}, + {'id': 3, 'data': '3'} + ] + + # write the assets + query.store_metadatas(conn, deepcopy(metadata)) + + # check that 3 assets were written to the database + cursor = conn.db.metadata.find({}, projection={'_id': False})\ + .sort('id', pymongo.ASCENDING) + + assert cursor.collection.count_documents({}) == 3 + assert list(cursor) == metadata + + +def test_get_metadata(): + from bigchaindb.backend import connect, query + conn = connect() + + metadata = [ + {'id': 1, 'metadata': None}, + {'id': 2, 'metadata': {'key': 'value'}}, + {'id': 3, 'metadata': '3'}, + ] + + conn.db.metadata.insert_many(deepcopy(metadata), ordered=False) + + for meta in metadata: + assert query.get_metadata(conn, [meta['id']]) + + +def test_get_owned_ids(signed_create_tx, user_pk): + from bigchaindb.backend import connect, query + conn = connect() + + # insert a transaction + conn.db.transactions.insert_one(deepcopy(signed_create_tx.to_dict())) + + txns = list(query.get_owned_ids(conn, user_pk)) + + assert txns[0] == signed_create_tx.to_dict() + + +def test_get_spending_transactions(user_pk, user_sk): + from bigchaindb.backend import connect, query + from bigchaindb.models import Transaction + conn = connect() + + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 3) + tx1.sign([user_sk]) + inputs = tx1.to_inputs() + tx2 = Transaction.transfer([inputs[0]], out, tx1.id).sign([user_sk]) + tx3 = Transaction.transfer([inputs[1]], out, tx1.id).sign([user_sk]) + tx4 = Transaction.transfer([inputs[2]], out, tx1.id).sign([user_sk]) + txns = [deepcopy(tx.to_dict()) for tx in [tx1, tx2, tx3, tx4]] + conn.db.transactions.insert_many(txns) + + links = [inputs[0].fulfills.to_dict(), inputs[2].fulfills.to_dict()] + txns = list(query.get_spending_transactions(conn, links)) + + # tx3 not a member because input 1 not asked for + assert txns == [tx2.to_dict(), tx4.to_dict()] + + +def test_get_spending_transactions_multiple_inputs(): + from bigchaindb.backend import connect, query + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + conn = connect() + (alice_sk, alice_pk) = generate_key_pair() + (bob_sk, bob_pk) = generate_key_pair() + (carol_sk, carol_pk) = generate_key_pair() + + out = [([alice_pk], 9)] + tx1 = Transaction.create([alice_pk], out).sign([alice_sk]) + + inputs1 = tx1.to_inputs() + tx2 = Transaction.transfer([inputs1[0]], + [([alice_pk], 6), ([bob_pk], 3)], + tx1.id).sign([alice_sk]) + + inputs2 = tx2.to_inputs() + tx3 = Transaction.transfer([inputs2[0]], + [([bob_pk], 3), ([carol_pk], 3)], + tx1.id).sign([alice_sk]) + + inputs3 = tx3.to_inputs() + tx4 = Transaction.transfer([inputs2[1], inputs3[0]], + [([carol_pk], 6)], + tx1.id).sign([bob_sk]) + + txns = [deepcopy(tx.to_dict()) for tx in [tx1, tx2, tx3, tx4]] + conn.db.transactions.insert_many(txns) + + links = [ + ({'transaction_id': tx2.id, 'output_index': 0}, 1, [tx3.id]), + ({'transaction_id': tx2.id, 'output_index': 1}, 1, [tx4.id]), + ({'transaction_id': tx3.id, 'output_index': 0}, 1, [tx4.id]), + ({'transaction_id': tx3.id, 'output_index': 1}, 0, None), + ] + for li, num, match in links: + txns = list(query.get_spending_transactions(conn, [li])) + assert len(txns) == num + if len(txns): + assert [tx['id'] for tx in txns] == match + + +def test_store_block(): + from bigchaindb.backend import connect, query + from bigchaindb.lib import Block + conn = connect() + + block = Block(app_hash='random_utxo', + height=3, + transactions=[]) + query.store_block(conn, block._asdict()) + cursor = conn.db.blocks.find({}, projection={'_id': False}) + assert cursor.collection.count_documents({}) == 1 + + +def test_get_block(): + from bigchaindb.backend import connect, query + from bigchaindb.lib import Block + conn = connect() + + block = Block(app_hash='random_utxo', + height=3, + transactions=[]) + + conn.db.blocks.insert_one(block._asdict()) + + block = dict(query.get_block(conn, 3)) + assert block['height'] == 3 + + +def test_delete_zero_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + unspent_outputs, utxo_collection = utxoset + delete_res = query.delete_unspent_outputs(db_context.conn) + assert delete_res is None + assert utxo_collection.count_documents({}) == 3 + assert utxo_collection.count_documents( + {'$or': [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'b', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 1}, + ]} + ) == 3 + + +def test_delete_one_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + unspent_outputs, utxo_collection = utxoset + delete_res = query.delete_unspent_outputs(db_context.conn, + unspent_outputs[0]) + assert delete_res.raw_result['n'] == 1 + assert utxo_collection.count_documents( + {'$or': [ + {'transaction_id': 'a', 'output_index': 1}, + {'transaction_id': 'b', 'output_index': 0}, + ]} + ) == 2 + assert utxo_collection.count_documents( + {'transaction_id': 'a', 'output_index': 0}) == 0 + + +def test_delete_many_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + unspent_outputs, utxo_collection = utxoset + delete_res = query.delete_unspent_outputs(db_context.conn, + *unspent_outputs[::2]) + assert delete_res.raw_result['n'] == 2 + assert utxo_collection.count_documents( + {'$or': [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'b', 'output_index': 0}, + ]} + ) == 0 + assert utxo_collection.count_documents( + {'transaction_id': 'a', 'output_index': 1}) == 1 + + +def test_store_zero_unspent_output(db_context, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn) + assert res is None + assert utxo_collection.count_documents({}) == 0 + + +def test_store_one_unspent_output(db_context, + unspent_output_1, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn, unspent_output_1) + assert res.acknowledged + assert len(res.inserted_ids) == 1 + assert utxo_collection.count_documents( + {'transaction_id': unspent_output_1['transaction_id'], + 'output_index': unspent_output_1['output_index']} + ) == 1 + + +def test_store_many_unspent_outputs(db_context, + unspent_outputs, utxo_collection): + from bigchaindb.backend import query + res = query.store_unspent_outputs(db_context.conn, *unspent_outputs) + assert res.acknowledged + assert len(res.inserted_ids) == 3 + assert utxo_collection.count_documents( + {'transaction_id': unspent_outputs[0]['transaction_id']} + ) == 3 + + +def test_get_unspent_outputs(db_context, utxoset): + from bigchaindb.backend import query + cursor = query.get_unspent_outputs(db_context.conn) + assert cursor.collection.count_documents({}) == 3 + retrieved_utxoset = list(cursor) + unspent_outputs, utxo_collection = utxoset + assert retrieved_utxoset == list( + utxo_collection.find(projection={'_id': False})) + assert retrieved_utxoset == unspent_outputs + + +def test_store_pre_commit_state(db_context): + from bigchaindb.backend import query + + state = dict(height=3, transactions=[]) + + query.store_pre_commit_state(db_context.conn, state) + cursor = db_context.conn.db.pre_commit.find({'commit_id': 'test'}, + projection={'_id': False}) + assert cursor.collection.count_documents({}) == 1 + + +def test_get_pre_commit_state(db_context): + from bigchaindb.backend import query + + state = dict(height=3, transactions=[]) + db_context.conn.db.pre_commit.insert_one(state) + resp = query.get_pre_commit_state(db_context.conn) + assert resp == state + + +def test_validator_update(): + from bigchaindb.backend import connect, query + + conn = connect() + + def gen_validator_update(height): + return {'data': 'somedata', 'height': height, 'election_id': f'election_id_at_height_{height}'} + + for i in range(1, 100, 10): + value = gen_validator_update(i) + query.store_validator_set(conn, value) + + v1 = query.get_validator_set(conn, 8) + assert v1['height'] == 1 + + v41 = query.get_validator_set(conn, 50) + assert v41['height'] == 41 + + v91 = query.get_validator_set(conn) + assert v91['height'] == 91 + + +@pytest.mark.parametrize('description,stores,expected', [ + ( + 'Query empty database.', + [], + None, + ), + ( + 'Store one chain with the default value for `is_synced`.', + [ + {'height': 0, 'chain_id': 'some-id'}, + ], + {'height': 0, 'chain_id': 'some-id', 'is_synced': True}, + ), + ( + 'Store one chain with a custom value for `is_synced`.', + [ + {'height': 0, 'chain_id': 'some-id', 'is_synced': False}, + ], + {'height': 0, 'chain_id': 'some-id', 'is_synced': False}, + ), + ( + 'Store one chain, then update it.', + [ + {'height': 0, 'chain_id': 'some-id', 'is_synced': True}, + {'height': 0, 'chain_id': 'new-id', 'is_synced': False}, + ], + {'height': 0, 'chain_id': 'new-id', 'is_synced': False}, + ), + ( + 'Store a chain, update it, store another chain.', + [ + {'height': 0, 'chain_id': 'some-id', 'is_synced': True}, + {'height': 0, 'chain_id': 'some-id', 'is_synced': False}, + {'height': 10, 'chain_id': 'another-id', 'is_synced': True}, + ], + {'height': 10, 'chain_id': 'another-id', 'is_synced': True}, + ), +]) +def test_store_abci_chain(description, stores, expected): + conn = connect() + + for store in stores: + query.store_abci_chain(conn, **store) + + actual = query.get_latest_abci_chain(conn) + assert expected == actual, description diff --git a/tests/backend/localmongodb/test_schema.py b/tests/backend/localmongodb/test_schema.py new file mode 100644 index 0000000..23d6dcd --- /dev/null +++ b/tests/backend/localmongodb/test_schema.py @@ -0,0 +1,76 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +def test_init_database_is_graceful_if_db_exists(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend.schema import init_database + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures + assert dbname in conn.conn.list_database_names() + + init_database() + + +def test_create_tables(): + import bigchaindb + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + dbname = bigchaindb.config['database']['name'] + + # The db is set up by the fixtures so we need to remove it + conn.conn.drop_database(dbname) + schema.create_database(conn, dbname) + schema.create_tables(conn, dbname) + + collection_names = conn.conn[dbname].list_collection_names() + assert set(collection_names) == { + 'transactions', 'assets', 'metadata', 'blocks', 'utxos', 'validators', 'elections', + 'pre_commit', 'abci_chains', + } + + indexes = conn.conn[dbname]['assets'].index_information().keys() + assert set(indexes) == {'_id_', 'asset_id', 'text'} + + index_info = conn.conn[dbname]['transactions'].index_information() + indexes = index_info.keys() + assert set(indexes) == { + '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} + assert index_info['transaction_id']['unique'] + + index_info = conn.conn[dbname]['blocks'].index_information() + indexes = index_info.keys() + assert set(indexes) == {'_id_', 'height'} + assert index_info['height']['unique'] + + index_info = conn.conn[dbname]['utxos'].index_information() + assert set(index_info.keys()) == {'_id_', 'utxo'} + assert index_info['utxo']['unique'] + assert index_info['utxo']['key'] == [('transaction_id', 1), + ('output_index', 1)] + + indexes = conn.conn[dbname]['elections'].index_information() + assert set(indexes.keys()) == {'_id_', 'election_id_height'} + assert indexes['election_id_height']['unique'] + + indexes = conn.conn[dbname]['pre_commit'].index_information() + assert set(indexes.keys()) == {'_id_', 'height'} + assert indexes['height']['unique'] + + +def test_drop(dummy_db): + from bigchaindb import backend + from bigchaindb.backend import schema + + conn = backend.connect() + assert dummy_db in conn.conn.list_database_names() + schema.drop_database(conn, dummy_db) + assert dummy_db not in conn.conn.list_database_names() diff --git a/tests/backend/test_connection.py b/tests/backend/test_connection.py new file mode 100644 index 0000000..a9c99c7 --- /dev/null +++ b/tests/backend/test_connection.py @@ -0,0 +1,22 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + + +def test_get_connection_raises_a_configuration_error(monkeypatch): + from bigchaindb.common.exceptions import ConfigurationError + from bigchaindb.backend import connect + + with pytest.raises(ConfigurationError): + connect('msaccess', 'localhost', '1337', 'mydb') + + with pytest.raises(ConfigurationError): + # We need to force a misconfiguration here + monkeypatch.setattr('bigchaindb.backend.connection.BACKENDS', + {'catsandra': + 'bigchaindb.backend.meowmeow.Catsandra'}) + + connect('catsandra', 'localhost', '1337', 'mydb') diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py new file mode 100644 index 0000000..684a93c --- /dev/null +++ b/tests/backend/test_generics.py @@ -0,0 +1,37 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from pytest import mark, raises + + +@mark.parametrize('schema_func_name,args_qty', ( + ('create_database', 1), + ('create_tables', 1), + ('drop_database', 1), +)) +def test_schema(schema_func_name, args_qty): + from bigchaindb.backend import schema + schema_func = getattr(schema, schema_func_name) + with raises(NotImplementedError): + schema_func(None, *range(args_qty)) + + +@mark.parametrize('query_func_name,args_qty', ( + ('delete_transactions', 1), + ('get_txids_filtered', 1), + ('get_owned_ids', 1), + ('get_block', 1), + ('get_spent', 2), + ('get_spending_transactions', 1), + ('store_assets', 1), + ('get_asset', 1), + ('store_metadatas', 1), + ('get_metadata', 1), +)) +def test_query(query_func_name, args_qty): + from bigchaindb.backend import query + query_func = getattr(query, query_func_name) + with raises(NotImplementedError): + query_func(None, *range(args_qty)) diff --git a/tests/backend/test_utils.py b/tests/backend/test_utils.py new file mode 100644 index 0000000..c720165 --- /dev/null +++ b/tests/backend/test_utils.py @@ -0,0 +1,77 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from functools import singledispatch +from types import ModuleType + +import pytest + + +@pytest.fixture +def mock_module(): + return ModuleType('mock_module') + + +def test_module_dispatch_registers(mock_module): + from bigchaindb.backend.utils import module_dispatch_registrar + + @singledispatch + def dispatcher(t): + pass + mock_module.dispatched = dispatcher + mock_dispatch = module_dispatch_registrar(mock_module) + + @mock_dispatch(str) + def dispatched(t): + pass + + assert mock_module.dispatched.registry[str] == dispatched + + +def test_module_dispatch_dispatches(mock_module): + from bigchaindb.backend.utils import module_dispatch_registrar + + @singledispatch + def dispatcher(t): + return False + mock_module.dispatched = dispatcher + mock_dispatch = module_dispatch_registrar(mock_module) + + @mock_dispatch(str) + def dispatched(t): + return True + + assert mock_module.dispatched(1) is False # Goes to dispatcher() + assert mock_module.dispatched('1') is True # Goes to dispatched() + + +def test_module_dispatch_errors_on_missing_func(mock_module): + from bigchaindb.backend.utils import ( + module_dispatch_registrar, + ModuleDispatchRegistrationError, + ) + mock_dispatch = module_dispatch_registrar(mock_module) + + with pytest.raises(ModuleDispatchRegistrationError): + @mock_dispatch(str) + def dispatched(): + pass + + +def test_module_dispatch_errors_on_non_dispatchable_func(mock_module): + from bigchaindb.backend.utils import ( + module_dispatch_registrar, + ModuleDispatchRegistrationError, + ) + + def dispatcher(): + pass + mock_module.dispatched = dispatcher + mock_dispatch = module_dispatch_registrar(mock_module) + + with pytest.raises(ModuleDispatchRegistrationError): + @mock_dispatch(str) + def dispatched(): + pass diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py new file mode 100644 index 0000000..0f60297 --- /dev/null +++ b/tests/commands/conftest.py @@ -0,0 +1,63 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from argparse import Namespace + +import pytest + + +@pytest.fixture +def mock_run_configure(monkeypatch): + from bigchaindb.commands import bigchaindb + monkeypatch.setattr(bigchaindb, 'run_configure', lambda *args, **kwargs: None) + + +@pytest.fixture +def mock_write_config(monkeypatch): + from bigchaindb import config_utils + monkeypatch.setattr(config_utils, 'write_config', lambda *args: None) + + +@pytest.fixture +def mock_db_init_with_existing_db(monkeypatch): + from bigchaindb.commands import bigchaindb + monkeypatch.setattr(bigchaindb, '_run_init', lambda: None) + + +@pytest.fixture +def mock_processes_start(monkeypatch): + from bigchaindb import start + monkeypatch.setattr(start, 'start', lambda *args: None) + + +@pytest.fixture +def mock_generate_key_pair(monkeypatch): + monkeypatch.setattr('bigchaindb.common.crypto.generate_key_pair', lambda: ('privkey', 'pubkey')) + + +@pytest.fixture +def mock_bigchaindb_backup_config(monkeypatch): + config = { + 'database': {'host': 'host', 'port': 12345, 'name': 'adbname'}, + } + monkeypatch.setattr('bigchaindb._config', config) + + +@pytest.fixture +def run_start_args(request): + param = getattr(request, 'param', {}) + return Namespace( + config=param.get('config'), + skip_initialize_database=param.get('skip_initialize_database', False), + ) + + +@pytest.fixture +def mocked_setup_logging(mocker): + return mocker.patch( + 'bigchaindb.log.setup_logging', + autospec=True, + spec_set=True, + ) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py new file mode 100644 index 0000000..2834473 --- /dev/null +++ b/tests/commands/test_commands.py @@ -0,0 +1,639 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import json +import logging + +from unittest.mock import Mock, patch +from argparse import Namespace + +import pytest + +from bigchaindb import ValidatorElection +from bigchaindb.commands.bigchaindb import run_election_show +from bigchaindb.elections.election import Election +from bigchaindb.lib import Block +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection + +from tests.utils import generate_election, generate_validators + + +def test_make_sure_we_dont_remove_any_command(): + # thanks to: http://stackoverflow.com/a/18161115/597097 + from bigchaindb.commands.bigchaindb import create_parser + + parser = create_parser() + + assert parser.parse_args(['configure', 'localmongodb']).command + assert parser.parse_args(['show-config']).command + assert parser.parse_args(['init']).command + assert parser.parse_args(['drop']).command + assert parser.parse_args(['start']).command + assert parser.parse_args(['election', 'new', 'upsert-validator', 'TEMP_PUB_KEYPAIR', '10', 'TEMP_NODE_ID', + '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command + assert parser.parse_args(['election', 'new', 'chain-migration', + '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command + assert parser.parse_args(['election', 'approve', 'ELECTION_ID', '--private-key', + 'TEMP_PATH_TO_PRIVATE_KEY']).command + assert parser.parse_args(['election', 'show', 'ELECTION_ID']).command + assert parser.parse_args(['tendermint-version']).command + + +@patch('bigchaindb.commands.utils.start') +def test_main_entrypoint(mock_start): + from bigchaindb.commands.bigchaindb import main + main() + + assert mock_start.called + + +@patch('bigchaindb.log.setup_logging') +@patch('bigchaindb.commands.bigchaindb._run_init') +@patch('bigchaindb.config_utils.autoconfigure') +def test_bigchain_run_start(mock_setup_logging, mock_run_init, + mock_autoconfigure, mock_processes_start): + from bigchaindb.commands.bigchaindb import run_start + args = Namespace(config=None, yes=True, + skip_initialize_database=False) + run_start(args) + assert mock_setup_logging.called + + +# TODO Please beware, that if debugging, the "-s" switch for pytest will +# interfere with capsys. +# See related issue: https://github.com/pytest-dev/pytest/issues/128 +@pytest.mark.usefixtures('ignore_local_config_file') +def test_bigchain_show_config(capsys): + from bigchaindb.commands.bigchaindb import run_show_config + + args = Namespace(config=None) + _, _ = capsys.readouterr() + run_show_config(args) + output_config = json.loads(capsys.readouterr()[0]) + # Note: This test passed previously because we were always + # using the default configuration parameters, but since we + # are running with docker-compose now and expose parameters like + # PLANETMINT_SERVER_BIND, PLANETMINT_WSSERVER_HOST, PLANETMINT_WSSERVER_ADVERTISED_HOST + # the default comparison fails i.e. when config is imported at the beginning the + # dict returned is different that what is expected after run_show_config + # and run_show_config updates the bigchaindb.config + from bigchaindb import config + del config['CONFIGURED'] + assert output_config == config + + +def test__run_init(mocker): + from bigchaindb.commands.bigchaindb import _run_init + bigchain_mock = mocker.patch( + 'bigchaindb.commands.bigchaindb.bigchaindb.Planetmint') + init_db_mock = mocker.patch( + 'bigchaindb.commands.bigchaindb.schema.init_database', + autospec=True, + spec_set=True, + ) + _run_init() + bigchain_mock.assert_called_once_with() + init_db_mock.assert_called_once_with( + connection=bigchain_mock.return_value.connection) + + +@patch('bigchaindb.backend.schema.drop_database') +def test_drop_db_when_assumed_yes(mock_db_drop): + from bigchaindb.commands.bigchaindb import run_drop + args = Namespace(config=None, yes=True) + + run_drop(args) + assert mock_db_drop.called + + +@patch('bigchaindb.backend.schema.drop_database') +def test_drop_db_when_interactive_yes(mock_db_drop, monkeypatch): + from bigchaindb.commands.bigchaindb import run_drop + args = Namespace(config=None, yes=False) + monkeypatch.setattr( + 'bigchaindb.commands.bigchaindb.input_on_stderr', lambda x: 'y') + + run_drop(args) + assert mock_db_drop.called + + +@patch('bigchaindb.backend.schema.drop_database') +def test_drop_db_when_db_does_not_exist(mock_db_drop, capsys): + from bigchaindb import config + from bigchaindb.commands.bigchaindb import run_drop + from bigchaindb.common.exceptions import DatabaseDoesNotExist + args = Namespace(config=None, yes=True) + mock_db_drop.side_effect = DatabaseDoesNotExist + + run_drop(args) + output_message = capsys.readouterr()[1] + assert output_message == "Cannot drop '{name}'. The database does not exist.\n".format( + name=config['database']['name']) + + +@patch('bigchaindb.backend.schema.drop_database') +def test_drop_db_does_not_drop_when_interactive_no(mock_db_drop, monkeypatch): + from bigchaindb.commands.bigchaindb import run_drop + args = Namespace(config=None, yes=False) + monkeypatch.setattr( + 'bigchaindb.commands.bigchaindb.input_on_stderr', lambda x: 'n') + + run_drop(args) + assert not mock_db_drop.called + + +# TODO Beware if you are putting breakpoints in there, and using the '-s' +# switch with pytest. It will just hang. Seems related to the monkeypatching of +# input_on_stderr. +def test_run_configure_when_config_does_not_exist(monkeypatch, + mock_write_config, + mock_generate_key_pair, + mock_bigchaindb_backup_config): + from bigchaindb.commands.bigchaindb import run_configure + monkeypatch.setattr('os.path.exists', lambda path: False) + monkeypatch.setattr('builtins.input', lambda: '\n') + args = Namespace(config=None, backend='localmongodb', yes=True) + return_value = run_configure(args) + assert return_value is None + + +def test_run_configure_when_config_does_exist(monkeypatch, + mock_write_config, + mock_generate_key_pair, + mock_bigchaindb_backup_config): + value = {} + + def mock_write_config(newconfig): + value['return'] = newconfig + + from bigchaindb.commands.bigchaindb import run_configure + monkeypatch.setattr('os.path.exists', lambda path: True) + monkeypatch.setattr('builtins.input', lambda: '\n') + monkeypatch.setattr( + 'bigchaindb.config_utils.write_config', mock_write_config) + + args = Namespace(config=None, yes=None) + run_configure(args) + assert value == {} + + +@pytest.mark.skip +@pytest.mark.parametrize('backend', ( + 'localmongodb', +)) +def test_run_configure_with_backend(backend, monkeypatch, mock_write_config): + import bigchaindb + from bigchaindb.commands.bigchaindb import run_configure + + value = {} + + def mock_write_config(new_config, filename=None): + value['return'] = new_config + + monkeypatch.setattr('os.path.exists', lambda path: False) + monkeypatch.setattr('builtins.input', lambda: '\n') + monkeypatch.setattr('bigchaindb.config_utils.write_config', + mock_write_config) + + args = Namespace(config=None, backend=backend, yes=True) + expected_config = bigchaindb.config + run_configure(args) + + # update the expected config with the correct backend and keypair + backend_conf = getattr(bigchaindb, '_database_' + backend) + expected_config.update({'database': backend_conf, + 'keypair': value['return']['keypair']}) + + assert value['return'] == expected_config + + +@patch('bigchaindb.commands.utils.start') +def test_calling_main(start_mock, monkeypatch): + from bigchaindb.commands.bigchaindb import main + + argparser_mock = Mock() + parser = Mock() + subparsers = Mock() + subsubparsers = Mock() + subparsers.add_parser.return_value = subsubparsers + parser.add_subparsers.return_value = subparsers + argparser_mock.return_value = parser + monkeypatch.setattr('argparse.ArgumentParser', argparser_mock) + main() + + assert argparser_mock.called is True + parser.add_subparsers.assert_called_with(title='Commands', + dest='command') + subparsers.add_parser.assert_any_call('configure', + help='Prepare the config file.') + subparsers.add_parser.assert_any_call('show-config', + help='Show the current ' + 'configuration') + subparsers.add_parser.assert_any_call('init', help='Init the database') + subparsers.add_parser.assert_any_call('drop', help='Drop the database') + + subparsers.add_parser.assert_any_call('start', help='Start Planetmint') + subparsers.add_parser.assert_any_call('tendermint-version', + help='Show the Tendermint supported ' + 'versions') + + assert start_mock.called is True + + +@patch('bigchaindb.commands.bigchaindb.run_recover') +@patch('bigchaindb.start.start') +def test_recover_db_on_start(mock_run_recover, + mock_start, + mocked_setup_logging): + from bigchaindb.commands.bigchaindb import run_start + args = Namespace(config=None, yes=True, + skip_initialize_database=False) + run_start(args) + + assert mock_run_recover.called + assert mock_start.called + + +@pytest.mark.bdb +def test_run_recover(b, alice, bob): + from bigchaindb.commands.bigchaindb import run_recover + from bigchaindb.models import Transaction + from bigchaindb.lib import Block + from bigchaindb.backend import query + + tx1 = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset={'cycle': 'hero'}, + metadata={'name': 'hohenheim'}) \ + .sign([alice.private_key]) + tx2 = Transaction.create([bob.public_key], + [([bob.public_key], 1)], + asset={'cycle': 'hero'}, + metadata={'name': 'hohenheim'}) \ + .sign([bob.private_key]) + + # store the transactions + b.store_bulk_transactions([tx1, tx2]) + + # create a random block + block8 = Block(app_hash='random_app_hash1', height=8, + transactions=['txid_doesnt_matter'])._asdict() + b.store_block(block8) + + # create the next block + block9 = Block(app_hash='random_app_hash1', height=9, + transactions=[tx1.id])._asdict() + b.store_block(block9) + + # create a pre_commit state which is ahead of the commit state + pre_commit_state = dict(height=10, transactions=[tx2.id]) + b.store_pre_commit_state(pre_commit_state) + + run_recover(b) + + assert not query.get_transaction(b.connection, tx2.id) + + +# Helper +class MockResponse(): + + def __init__(self, height): + self.height = height + + def json(self): + return {'result': {'latest_block_height': self.height}} + + +@pytest.mark.abci +def test_election_new_upsert_validator_with_tendermint(b, priv_validator_path, user_sk, validators): + from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator + + new_args = Namespace(action='new', + election_type='upsert-validator', + public_key='HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52GnGuPY=', + power=1, + node_id='unique_node_id_for_test_upsert_validator_new_with_tendermint', + sk=priv_validator_path, + config={}) + + election_id = run_election_new_upsert_validator(new_args, b) + + assert b.get_transaction(election_id) + + +@pytest.mark.bdb +def test_election_new_upsert_validator_without_tendermint(caplog, b, priv_validator_path, user_sk): + from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator + + def mock_write(tx, mode): + b.store_bulk_transactions([tx]) + return (202, '') + + b.get_validators = mock_get_validators + b.write_transaction = mock_write + + args = Namespace(action='new', + election_type='upsert-validator', + public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', + power=1, + node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', + sk=priv_validator_path, + config={}) + + with caplog.at_level(logging.INFO): + election_id = run_election_new_upsert_validator(args, b) + assert caplog.records[0].msg == '[SUCCESS] Submitted proposal with id: ' + election_id + assert b.get_transaction(election_id) + + +@pytest.mark.abci +def test_election_new_chain_migration_with_tendermint(b, priv_validator_path, user_sk, validators): + from bigchaindb.commands.bigchaindb import run_election_new_chain_migration + + new_args = Namespace(action='new', + election_type='migration', + sk=priv_validator_path, + config={}) + + election_id = run_election_new_chain_migration(new_args, b) + + assert b.get_transaction(election_id) + + +@pytest.mark.bdb +def test_election_new_chain_migration_without_tendermint(caplog, b, priv_validator_path, user_sk): + from bigchaindb.commands.bigchaindb import run_election_new_chain_migration + + def mock_write(tx, mode): + b.store_bulk_transactions([tx]) + return (202, '') + + b.get_validators = mock_get_validators + b.write_transaction = mock_write + + args = Namespace(action='new', + election_type='migration', + sk=priv_validator_path, + config={}) + + with caplog.at_level(logging.INFO): + election_id = run_election_new_chain_migration(args, b) + assert caplog.records[0].msg == '[SUCCESS] Submitted proposal with id: ' + election_id + assert b.get_transaction(election_id) + + +@pytest.mark.bdb +def test_election_new_upsert_validator_invalid_election(caplog, b, priv_validator_path, user_sk): + from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator + + args = Namespace(action='new', + election_type='upsert-validator', + public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', + power=10, + node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', + sk='/tmp/invalid/path/key.json', + config={}) + + with caplog.at_level(logging.ERROR): + assert not run_election_new_upsert_validator(args, b) + assert caplog.records[0].msg.__class__ == FileNotFoundError + + +@pytest.mark.bdb +def test_election_new_upsert_validator_invalid_power(caplog, b, priv_validator_path, user_sk): + from bigchaindb.commands.bigchaindb import run_election_new_upsert_validator + from bigchaindb.common.exceptions import InvalidPowerChange + + def mock_write(tx, mode): + b.store_bulk_transactions([tx]) + return (400, '') + + b.write_transaction = mock_write + b.get_validators = mock_get_validators + args = Namespace(action='new', + election_type='upsert-validator', + public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=', + power=10, + node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', + sk=priv_validator_path, + config={}) + + with caplog.at_level(logging.ERROR): + assert not run_election_new_upsert_validator(args, b) + assert caplog.records[0].msg.__class__ == InvalidPowerChange + + +@pytest.mark.abci +def test_election_approve_with_tendermint(b, priv_validator_path, user_sk, validators): + from bigchaindb.commands.bigchaindb import (run_election_new_upsert_validator, + run_election_approve) + + public_key = 'CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=' + new_args = Namespace(action='new', + election_type='upsert-validator', + public_key=public_key, + power=1, + node_id='fb7140f03a4ffad899fabbbf655b97e0321add66', + sk=priv_validator_path, + config={}) + + election_id = run_election_new_upsert_validator(new_args, b) + assert election_id + + args = Namespace(action='approve', + election_id=election_id, + sk=priv_validator_path, + config={}) + approve = run_election_approve(args, b) + + assert b.get_transaction(approve) + + +@pytest.mark.bdb +def test_election_approve_without_tendermint(caplog, b, priv_validator_path, new_validator, node_key): + from bigchaindb.commands.bigchaindb import run_election_approve + from argparse import Namespace + + b, election_id = call_election(b, new_validator, node_key) + + # call run_election_approve with args that point to the election + args = Namespace(action='approve', + election_id=election_id, + sk=priv_validator_path, + config={}) + + # assert returned id is in the db + with caplog.at_level(logging.INFO): + approval_id = run_election_approve(args, b) + assert caplog.records[0].msg == '[SUCCESS] Your vote has been submitted' + assert b.get_transaction(approval_id) + + +@pytest.mark.bdb +def test_election_approve_failure(caplog, b, priv_validator_path, new_validator, node_key): + from bigchaindb.commands.bigchaindb import run_election_approve + from argparse import Namespace + + b, election_id = call_election(b, new_validator, node_key) + + def mock_write(tx, mode): + b.store_bulk_transactions([tx]) + return (400, '') + + b.write_transaction = mock_write + + # call run_upsert_validator_approve with args that point to the election + args = Namespace(action='approve', + election_id=election_id, + sk=priv_validator_path, + config={}) + + with caplog.at_level(logging.ERROR): + assert not run_election_approve(args, b) + assert caplog.records[0].msg == 'Failed to commit vote' + + +@pytest.mark.bdb +def test_election_approve_called_with_bad_key(caplog, b, bad_validator_path, new_validator, node_key): + from bigchaindb.commands.bigchaindb import run_election_approve + from argparse import Namespace + + b, election_id = call_election(b, new_validator, node_key) + + # call run_upsert_validator_approve with args that point to the election, but a bad signing key + args = Namespace(action='approve', + election_id=election_id, + sk=bad_validator_path, + config={}) + + with caplog.at_level(logging.ERROR): + assert not run_election_approve(args, b) + assert caplog.records[0].msg == 'The key you provided does not match any of '\ + 'the eligible voters in this election.' + + +@pytest.mark.bdb +def test_chain_migration_election_show_shows_inconclusive(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + assert not run_election_show(Namespace(election_id=election.id), b) + + Election.process_block(b, 1, [election]) + b.store_bulk_transactions([election]) + + assert run_election_show(Namespace(election_id=election.id), b) == \ + 'status=ongoing' + + b.store_block(Block(height=1, transactions=[], app_hash='')._asdict()) + b.store_validator_set(2, [v['storage'] for v in validators]) + + assert run_election_show(Namespace(election_id=election.id), b) == \ + 'status=ongoing' + + b.store_block(Block(height=2, transactions=[], app_hash='')._asdict()) + # TODO insert yet another block here when upgrading to Tendermint 0.22.4. + + assert run_election_show(Namespace(election_id=election.id), b) == \ + 'status=inconclusive' + + +@pytest.mark.bdb +def test_chain_migration_election_show_shows_concluded(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + assert not run_election_show(Namespace(election_id=election.id), b) + + b.store_bulk_transactions([election]) + Election.process_block(b, 1, [election]) + + assert run_election_show(Namespace(election_id=election.id), b) == \ + 'status=ongoing' + + b.store_abci_chain(1, 'chain-X') + b.store_block(Block(height=1, + transactions=[v.id for v in votes], + app_hash='last_app_hash')._asdict()) + Election.process_block(b, 2, votes) + + assert run_election_show(Namespace(election_id=election.id), b) == \ + f'''status=concluded +chain_id=chain-X-migrated-at-height-1 +app_hash=last_app_hash +validators=[{''.join([f""" + {{ + "pub_key": {{ + "type": "tendermint/PubKeyEd25519", + "value": "{v['public_key']}" + }}, + "power": {v['storage']['voting_power']} + }}{',' if i + 1 != len(validators) else ''}""" for i, v in enumerate(validators)])} +]''' + + +def test_bigchain_tendermint_version(capsys): + from bigchaindb.commands.bigchaindb import run_tendermint_version + + args = Namespace(config=None) + _, _ = capsys.readouterr() + run_tendermint_version(args) + output_config = json.loads(capsys.readouterr()[0]) + from bigchaindb.version import __tm_supported_versions__ + assert len(output_config["tendermint"]) == len(__tm_supported_versions__) + assert sorted(output_config["tendermint"]) == sorted(__tm_supported_versions__) + + +def mock_get_validators(height): + return [ + {'public_key': {'value': "zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=", + 'type': 'ed25519-base64'}, + 'voting_power': 10} + ] + + +def call_election(b, new_validator, node_key): + + def mock_write(tx, mode): + b.store_bulk_transactions([tx]) + return (202, '') + + # patch the validator set. We now have one validator with power 10 + b.get_validators = mock_get_validators + b.write_transaction = mock_write + + # our voters is a list of length 1, populated from our mocked validator + voters = ValidatorElection.recipients(b) + # and our voter is the public key from the voter list + voter = node_key.public_key + valid_election = ValidatorElection.generate([voter], + voters, + new_validator, None).sign([node_key.private_key]) + + # patch in an election with a vote issued to the user + election_id = valid_election.id + b.store_bulk_transactions([valid_election]) + + return b, election_id diff --git a/tests/commands/test_utils.py b/tests/commands/test_utils.py new file mode 100644 index 0000000..f298161 --- /dev/null +++ b/tests/commands/test_utils.py @@ -0,0 +1,133 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import argparse +from argparse import Namespace +import logging + +import pytest + +from unittest.mock import patch + + +@pytest.fixture +def reset_bigchaindb_config(monkeypatch): + import bigchaindb + monkeypatch.setattr('bigchaindb.config', bigchaindb._config) + + +def test_input_on_stderr(): + from bigchaindb.commands.utils import input_on_stderr, _convert + + with patch('builtins.input', return_value='I love cats'): + assert input_on_stderr() == 'I love cats' + + # input_on_stderr uses `_convert` internally, from now on we will + # just use that function + + assert _convert('hack the planet') == 'hack the planet' + assert _convert('42') == '42' + assert _convert('42', default=10) == 42 + assert _convert('', default=10) == 10 + assert _convert('42', convert=int) == 42 + assert _convert('True', convert=bool) is True + assert _convert('False', convert=bool) is False + assert _convert('t', convert=bool) is True + assert _convert('3.14', default=1.0) == 3.14 + assert _convert('TrUe', default=False) is True + + with pytest.raises(ValueError): + assert _convert('TRVE', default=False) + + with pytest.raises(ValueError): + assert _convert('ಠ_ಠ', convert=int) + + +@pytest.mark.usefixtures('ignore_local_config_file', 'reset_bigchaindb_config') +def test_configure_bigchaindb_configures_bigchaindb(): + from bigchaindb.commands.utils import configure_bigchaindb + from bigchaindb.config_utils import is_configured + assert not is_configured() + + @configure_bigchaindb + def test_configure(args): + assert is_configured() + + args = Namespace(config=None) + test_configure(args) + + +@pytest.mark.usefixtures('ignore_local_config_file', + 'reset_bigchaindb_config', + 'reset_logging_config') +@pytest.mark.parametrize('log_level', tuple(map( + logging.getLevelName, + (logging.DEBUG, + logging.INFO, + logging.WARNING, + logging.ERROR, + logging.CRITICAL) +))) +def test_configure_bigchaindb_logging(log_level): + # TODO: See following comment: + # This is a dirty test. If a test *preceding* this test makes use of the logger, and then another test *after* this + # test also makes use of the logger, somehow we get logger.disabled == True, and the later test fails. We need to + # either engineer this somehow to leave the test env in the same state as it finds it, or make an assessment + # whether or not we even need this test, and potentially just remove it. + + from bigchaindb.commands.utils import configure_bigchaindb + + @configure_bigchaindb + def test_configure_logger(args): + pass + + args = Namespace(config=None, log_level=log_level) + test_configure_logger(args) + from bigchaindb import config + assert config['log']['level_console'] == log_level + assert config['log']['level_logfile'] == log_level + + +def test_start_raises_if_command_not_implemented(): + from bigchaindb.commands import utils + from bigchaindb.commands.bigchaindb import create_parser + + parser = create_parser() + + with pytest.raises(NotImplementedError): + # Will raise because `scope`, the third parameter, + # doesn't contain the function `run_start` + utils.start(parser, ['start'], {}) + + +def test_start_raises_if_no_arguments_given(): + from bigchaindb.commands import utils + from bigchaindb.commands.bigchaindb import create_parser + + parser = create_parser() + + with pytest.raises(SystemExit): + utils.start(parser, [], {}) + + +@patch('multiprocessing.cpu_count', return_value=42) +def test_start_sets_multiprocess_var_based_on_cli_args(mock_cpu_count): + from bigchaindb.commands import utils + + def run_mp_arg_test(args): + return args + + parser = argparse.ArgumentParser() + subparser = parser.add_subparsers(title='Commands', + dest='command') + mp_arg_test_parser = subparser.add_parser('mp_arg_test') + mp_arg_test_parser.add_argument('-m', '--multiprocess', + nargs='?', + type=int, + default=False) + + scope = {'run_mp_arg_test': run_mp_arg_test} + assert utils.start(parser, ['mp_arg_test'], scope).multiprocess == 1 + assert utils.start(parser, ['mp_arg_test', '--multiprocess'], scope).multiprocess == 42 diff --git a/tests/common/__init__.py b/tests/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common/conftest.py b/tests/common/conftest.py new file mode 100644 index 0000000..35b1fc0 --- /dev/null +++ b/tests/common/conftest.py @@ -0,0 +1,307 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from base58 import b58decode +import pytest + + +USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie' +USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' + +USER2_PRIVATE_KEY = 'F86PQPiqMTwM2Qi2Sda3U4Vdh3AgadMdX3KNVsu5wNJr' +USER2_PUBLIC_KEY = 'GDxwMFbwdATkQELZbMfW8bd9hbNYMZLyVXA3nur2aNbE' + +USER3_PRIVATE_KEY = '4rNQFzWQbVwuTiDVxwuFMvLG5zd8AhrQKCtVovBvcYsB' +USER3_PUBLIC_KEY = 'Gbrg7JtxdjedQRmr81ZZbh1BozS7fBW88ZyxNDy7WLNC' + +CC_FULFILLMENT_URI = ( + 'pGSAINdamAGCsQq31Uv-08lkBzoO4XLz2qYjJa8CGmj3B1EagUDlVkMAw2CscpCG4syAboKKh' + 'Id_Hrjl2XTYc-BlIkkBVV-4ghWQozusxh45cBz5tGvSW_XwWVu-JGVRQUOOehAL' +) +CC_CONDITION_URI = ('ni:///sha-256;' + 'eZI5q6j8T_fqv7xMROaei9_tmTMk4S7WR5Kr4onPHV8' + '?fpt=ed25519-sha-256&cost=131072') + +ASSET_DEFINITION = { + 'data': { + 'definition': 'Asset definition' + } +} + +DATA = { + 'msg': 'Hello Planetmint!' +} + + +@pytest.fixture +def user_priv(): + return USER_PRIVATE_KEY + + +@pytest.fixture +def user_pub(): + return USER_PUBLIC_KEY + + +@pytest.fixture +def user2_priv(): + return USER2_PRIVATE_KEY + + +@pytest.fixture +def user2_pub(): + return USER2_PUBLIC_KEY + + +@pytest.fixture +def user3_priv(): + return USER3_PRIVATE_KEY + + +@pytest.fixture +def user3_pub(): + return USER3_PUBLIC_KEY + + +@pytest.fixture +def ffill_uri(): + return CC_FULFILLMENT_URI + + +@pytest.fixture +def cond_uri(): + return CC_CONDITION_URI + + +@pytest.fixture +def user_Ed25519(user_pub): + from cryptoconditions import Ed25519Sha256 + return Ed25519Sha256(public_key=b58decode(user_pub)) + + +@pytest.fixture +def user_user2_threshold(user_pub, user2_pub): + from cryptoconditions import ThresholdSha256, Ed25519Sha256 + user_pub_keys = [user_pub, user2_pub] + threshold = ThresholdSha256(threshold=len(user_pub_keys)) + for user_pub in user_pub_keys: + threshold.add_subfulfillment( + Ed25519Sha256(public_key=b58decode(user_pub))) + return threshold + + +@pytest.fixture +def user2_Ed25519(user2_pub): + from cryptoconditions import Ed25519Sha256 + return Ed25519Sha256(public_key=b58decode(user2_pub)) + + +@pytest.fixture +def user_input(user_Ed25519, user_pub): + from bigchaindb.common.transaction import Input + return Input(user_Ed25519, [user_pub]) + + +@pytest.fixture +def user_user2_threshold_output(user_user2_threshold, user_pub, user2_pub): + from bigchaindb.common.transaction import Output + return Output(user_user2_threshold, [user_pub, user2_pub]) + + +@pytest.fixture +def user_user2_threshold_input(user_user2_threshold, user_pub, user2_pub): + from bigchaindb.common.transaction import Input + return Input(user_user2_threshold, [user_pub, user2_pub]) + + +@pytest.fixture +def user_output(user_Ed25519, user_pub): + from bigchaindb.common.transaction import Output + return Output(user_Ed25519, [user_pub]) + + +@pytest.fixture +def user2_output(user2_Ed25519, user2_pub): + from bigchaindb.common.transaction import Output + return Output(user2_Ed25519, [user2_pub]) + + +@pytest.fixture +def asset_definition(): + return ASSET_DEFINITION + + +@pytest.fixture +def data(): + return DATA + + +@pytest.fixture +def utx(user_input, user_output): + from bigchaindb.common.transaction import Transaction + return Transaction(Transaction.CREATE, {'data': None}, [user_input], + [user_output]) + + +@pytest.fixture +def tx(utx, user_priv): + return utx.sign([user_priv]) + + +@pytest.fixture +def transfer_utx(user_output, user2_output, utx): + from bigchaindb.common.transaction import (Input, TransactionLink, + Transaction) + user_output = user_output.to_dict() + input = Input(utx.outputs[0].fulfillment, + user_output['public_keys'], + TransactionLink(utx.id, 0)) + return Transaction('TRANSFER', {'id': utx.id}, [input], [user2_output]) + + +@pytest.fixture +def transfer_tx(transfer_utx, user_priv): + return transfer_utx.sign([user_priv]) + + +@pytest.fixture(scope="session") +def dummy_transaction(): + return { + 'asset': {'data': None}, + 'id': 64 * 'a', + 'inputs': [{ + 'fulfillment': 'dummy', + 'fulfills': None, + 'owners_before': [58 * 'a'], + }], + 'metadata': None, + 'operation': 'CREATE', + 'outputs': [{ + 'amount': '1', + 'condition': { + 'details': { + 'public_key': 58 * 'b', + 'type': 'ed25519-sha-256' + }, + 'uri': 'dummy', + }, + 'public_keys': [58 * 'b'] + }], + 'version': '2.0' + } + + +@pytest.fixture +def unfulfilled_transaction(): + return { + 'asset': { + 'data': { + 'msg': 'Hello Planetmint!', + } + }, + 'id': None, + 'inputs': [{ + # XXX This could be None, see #1925 + # https://github.com/bigchaindb/bigchaindb/issues/1925 + 'fulfillment': { + 'public_key': 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE', + 'type': 'ed25519-sha-256' + }, + 'fulfills': None, + 'owners_before': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] + }], + 'metadata': None, + 'operation': 'CREATE', + 'outputs': [{ + 'amount': '1', + 'condition': { + 'details': { + 'public_key': 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE', + 'type': 'ed25519-sha-256' + }, + 'uri': 'ni:///sha-256;49C5UWNODwtcINxLgLc90bMCFqCymFYONGEmV4a0sG4?fpt=ed25519-sha-256&cost=131072'}, + 'public_keys': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] + }], + 'version': '1.0' + } + + +@pytest.fixture +def fulfilled_transaction(): + return { + 'asset': { + 'data': { + 'msg': 'Hello Planetmint!', + } + }, + 'id': None, + 'inputs': [{ + 'fulfillment': ('pGSAIP_2P1Juh-94sD3uno1lxMPd9EkIalRo7QB014pT6dD9g' + 'UANRNxasDy1Dfg9C2Fk4UgHdYFsJzItVYi5JJ_vWc6rKltn0k' + 'jagynI0xfyR6X9NhzccTt5oiNH9mThEb4QmagN'), + 'fulfills': None, + 'owners_before': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] + }], + 'metadata': None, + 'operation': 'CREATE', + 'outputs': [{ + 'amount': '1', + 'condition': { + 'details': { + 'public_key': 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE', + 'type': 'ed25519-sha-256' + }, + 'uri': 'ni:///sha-256;49C5UWNODwtcINxLgLc90bMCFqCymFYONGEmV4a0sG4?fpt=ed25519-sha-256&cost=131072'}, + 'public_keys': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] + }], + 'version': '1.0' + } + + +# TODO For reviewers: Pick which approach you like best: parametrized or not? +@pytest.fixture(params=( + {'id': None, + 'fulfillment': { + 'public_key': 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE', + 'type': 'ed25519-sha-256'}}, + {'id': None, + 'fulfillment': ('pGSAIP_2P1Juh-94sD3uno1lxMPd9EkIalRo7QB014pT6dD9g' + 'UANRNxasDy1Dfg9C2Fk4UgHdYFsJzItVYi5JJ_vWc6rKltn0k' + 'jagynI0xfyR6X9NhzccTt5oiNH9mThEb4QmagN')}, + {'id': '7a7c827cf4ef7985f08f4e9d16f5ffc58ca4e82271921dfbed32e70cb462485f', + 'fulfillment': ('pGSAIP_2P1Juh-94sD3uno1lxMPd9EkIalRo7QB014pT6dD9g' + 'UANRNxasDy1Dfg9C2Fk4UgHdYFsJzItVYi5JJ_vWc6rKltn0k' + 'jagynI0xfyR6X9NhzccTt5oiNH9mThEb4QmagN')}, +)) +def tri_state_transaction(request): + tx = { + 'asset': { + 'data': { + 'msg': 'Hello Planetmint!', + } + }, + 'id': None, + 'inputs': [{ + 'fulfillment': None, + 'fulfills': None, + 'owners_before': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] + }], + 'metadata': None, + 'operation': 'CREATE', + 'outputs': [{ + 'amount': '1', + 'condition': { + 'details': { + 'public_key': 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE', + 'type': 'ed25519-sha-256' + }, + 'uri': 'ni:///sha-256;49C5UWNODwtcINxLgLc90bMCFqCymFYONGEmV4a0sG4?fpt=ed25519-sha-256&cost=131072'}, + 'public_keys': ['JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'] + }], + 'version': '2.0' + } + tx['id'] = request.param['id'] + tx['inputs'][0]['fulfillment'] = request.param['fulfillment'] + return tx diff --git a/tests/common/test_memoize.py b/tests/common/test_memoize.py new file mode 100644 index 0000000..3a224e7 --- /dev/null +++ b/tests/common/test_memoize.py @@ -0,0 +1,93 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest +from copy import deepcopy + +from bigchaindb.models import Transaction +from bigchaindb.common.crypto import generate_key_pair +from bigchaindb.common.memoize import to_dict, from_dict + + +pytestmark = pytest.mark.bdb + + +def test_memoize_to_dict(b): + alice = generate_key_pair() + asset = { + 'data': {'id': 'test_id'}, + } + + assert to_dict.cache_info().hits == 0 + assert to_dict.cache_info().misses == 0 + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=asset,)\ + .sign([alice.private_key]) + + tx.to_dict() + + assert to_dict.cache_info().hits == 0 + assert to_dict.cache_info().misses == 1 + + tx.to_dict() + tx.to_dict() + + assert to_dict.cache_info().hits == 2 + assert to_dict.cache_info().misses == 1 + + +def test_memoize_from_dict(b): + alice = generate_key_pair() + asset = { + 'data': {'id': 'test_id'}, + } + + assert from_dict.cache_info().hits == 0 + assert from_dict.cache_info().misses == 0 + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=asset,)\ + .sign([alice.private_key]) + tx_dict = deepcopy(tx.to_dict()) + + Transaction.from_dict(tx_dict) + + assert from_dict.cache_info().hits == 0 + assert from_dict.cache_info().misses == 1 + + Transaction.from_dict(tx_dict) + Transaction.from_dict(tx_dict) + + assert from_dict.cache_info().hits == 2 + assert from_dict.cache_info().misses == 1 + + +def test_memoize_input_valid(b): + alice = generate_key_pair() + asset = { + 'data': {'id': 'test_id'}, + } + + assert Transaction._input_valid.cache_info().hits == 0 + assert Transaction._input_valid.cache_info().misses == 0 + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=asset,)\ + .sign([alice.private_key]) + + tx.inputs_valid() + + assert Transaction._input_valid.cache_info().hits == 0 + assert Transaction._input_valid.cache_info().misses == 1 + + tx.inputs_valid() + tx.inputs_valid() + + assert Transaction._input_valid.cache_info().hits == 2 + assert Transaction._input_valid.cache_info().misses == 1 diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py new file mode 100644 index 0000000..b38b499 --- /dev/null +++ b/tests/common/test_schema.py @@ -0,0 +1,131 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""This module is tests related to schema checking, but _not_ of granular schematic +properties related to validation. +""" + +from unittest.mock import patch + +from hypothesis import given +from hypothesis.strategies import from_regex as regex +from pytest import raises + +from bigchaindb.common.exceptions import SchemaValidationError +from bigchaindb.common.schema import ( + TX_SCHEMA_COMMON, + validate_transaction_schema, +) + +SUPPORTED_CRYPTOCONDITION_TYPES = ('threshold-sha-256', 'ed25519-sha-256') +UNSUPPORTED_CRYPTOCONDITION_TYPES = ( + 'preimage-sha-256', 'prefix-sha-256', 'rsa-sha-256') + + +################################################################################ +# Test of schema utils + + +def _test_additionalproperties(node, path=''): + """Validate that each object node has additionalProperties set, so that + objects with junk keys do not pass as valid. + """ + if isinstance(node, list): + for i, nnode in enumerate(node): + _test_additionalproperties(nnode, path + str(i) + '.') + if isinstance(node, dict): + if node.get('type') == 'object': + assert 'additionalProperties' in node, \ + ('additionalProperties not set at path:' + path) + for name, val in node.items(): + _test_additionalproperties(val, path + name + '.') + + +def test_transaction_schema_additionalproperties(): + _test_additionalproperties(TX_SCHEMA_COMMON) + + +################################################################################ +# Test call transaction schema + + +def test_validate_transaction_create(create_tx): + validate_transaction_schema(create_tx.to_dict()) + + +def test_validate_transaction_signed_create(signed_create_tx): + validate_transaction_schema(signed_create_tx.to_dict()) + + +def test_validate_transaction_signed_transfer(signed_transfer_tx): + validate_transaction_schema(signed_transfer_tx.to_dict()) + + +def test_validate_transaction_fails(): + with raises(SchemaValidationError): + validate_transaction_schema({}) + + +def test_validate_failure_inconsistent(): + with patch('jsonschema.validate'): + with raises(SchemaValidationError): + validate_transaction_schema({}) + + +@given(condition_uri=regex( + r'^ni:\/\/\/sha-256;([a-zA-Z0-9_-]{{0,86}})\?fpt=({})' + r'&cost=[0-9]+(?![\n])$'.format('|'.join( + t for t in SUPPORTED_CRYPTOCONDITION_TYPES)))) +def test_condition_uri_with_supported_fpt(dummy_transaction, condition_uri): + dummy_transaction['outputs'][0]['condition']['uri'] = condition_uri + validate_transaction_schema(dummy_transaction) + + +@given(condition_uri=regex(r'^ni:\/\/\/sha-256;([a-zA-Z0-9_-]{{0,86}})\?fpt=' + r'({})&cost=[0-9]+(?![\n])$'.format( + '|'.join(UNSUPPORTED_CRYPTOCONDITION_TYPES)))) +def test_condition_uri_with_unsupported_fpt(dummy_transaction, condition_uri): + dummy_transaction['outputs'][0]['condition']['uri'] = condition_uri + with raises(SchemaValidationError): + validate_transaction_schema(dummy_transaction) + + +@given(condition_uri=regex( + r'^ni:\/\/\/sha-256;([a-zA-Z0-9_-]{{0,86}})\?fpt=(?!{})' + r'&cost=[0-9]+(?![\n])$'.format('$|'.join( + t for t in SUPPORTED_CRYPTOCONDITION_TYPES)))) +def test_condition_uri_with_unknown_fpt(dummy_transaction, condition_uri): + dummy_transaction['outputs'][0]['condition']['uri'] = condition_uri + with raises(SchemaValidationError): + validate_transaction_schema(dummy_transaction) + + +@given(condition_uri=regex( + r'^ni:\/\/\/sha-256;([a-zA-Z0-9_-]{0,86})\?fpt=threshold-sha-256' + r'&cost=[0-9]+&subtypes=ed25519-sha-256(?![\n])$')) +def test_condition_uri_with_supported_subtype(dummy_transaction, + condition_uri): + dummy_transaction['outputs'][0]['condition']['uri'] = condition_uri + validate_transaction_schema(dummy_transaction) + + +@given(condition_uri=regex( + r'^ni:\/\/\/sha-256;([a-zA-Z0-9_-]{0,86})\?fpt=threshold-sha-256&cost=' + r'[0-9]+&subtypes=(preimage-sha-256|prefix-sha-256|rsa-sha-256)(?![\n])$')) +def test_condition_uri_with_unsupported_subtype(dummy_transaction, + condition_uri): + dummy_transaction['outputs'][0]['condition']['uri'] = condition_uri + with raises(SchemaValidationError): + validate_transaction_schema(dummy_transaction) + + +@given(condition_uri=regex( + r'^ni:\/\/\/sha-256;([a-zA-Z0-9_-]{{0,86}})\?fpt=threshold-sha-256' + r'&cost=[0-9]+&subtypes=(?!{})(?![\n])$'.format('$|'.join( + t for t in SUPPORTED_CRYPTOCONDITION_TYPES)))) +def test_condition_uri_with_unknown_subtype(dummy_transaction, condition_uri): + dummy_transaction['outputs'][0]['condition']['uri'] = condition_uri + with raises(SchemaValidationError): + validate_transaction_schema(dummy_transaction) diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py new file mode 100644 index 0000000..99afe5b --- /dev/null +++ b/tests/common/test_transaction.py @@ -0,0 +1,1057 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""These are tests of the API of the Transaction class and associated classes. +Tests for transaction validation are separate. +""" +import json +from copy import deepcopy + +from base58 import b58encode, b58decode +from cryptoconditions import Ed25519Sha256 +from pytest import mark, raises +try: + from hashlib import sha3_256 +except ImportError: + from sha3 import sha3_256 + +pytestmark = mark.bdb + + +def test_input_serialization(ffill_uri, user_pub): + from bigchaindb.common.transaction import Input + from cryptoconditions import Fulfillment + + expected = { + 'owners_before': [user_pub], + 'fulfillment': ffill_uri, + 'fulfills': None, + } + input = Input(Fulfillment.from_uri(ffill_uri), [user_pub]) + assert input.to_dict() == expected + + +def test_input_deserialization_with_uri(ffill_uri, user_pub): + from bigchaindb.common.transaction import Input + from cryptoconditions import Fulfillment + + expected = Input(Fulfillment.from_uri(ffill_uri), [user_pub]) + ffill = { + 'owners_before': [user_pub], + 'fulfillment': ffill_uri, + 'fulfills': None, + } + input = Input.from_dict(ffill) + + assert input == expected + + +@mark.skip(reason='None is tolerated because it is None before fulfilling.') +def test_input_deserialization_with_invalid_input(user_pub): + from bigchaindb.common.transaction import Input + + ffill = { + 'owners_before': [user_pub], + 'fulfillment': None, + 'fulfills': None, + } + with raises(TypeError): + Input.from_dict(ffill) + + +def test_input_deserialization_with_invalid_fulfillment_uri(user_pub): + from bigchaindb.common.exceptions import InvalidSignature + from bigchaindb.common.transaction import Input + + ffill = { + 'owners_before': [user_pub], + 'fulfillment': 'an invalid fulfillment', + 'fulfills': None, + } + with raises(InvalidSignature): + Input.from_dict(ffill) + + +def test_input_deserialization_with_unsigned_fulfillment(ffill_uri, user_pub): + from bigchaindb.common.transaction import Input + from cryptoconditions import Fulfillment + + expected = Input(Fulfillment.from_uri(ffill_uri), [user_pub]) + ffill = { + 'owners_before': [user_pub], + 'fulfillment': Fulfillment.from_uri(ffill_uri), + 'fulfills': None, + } + input = Input.from_dict(ffill) + + assert input == expected + + +def test_output_serialization(user_Ed25519, user_pub): + from bigchaindb.common.transaction import Output + + expected = { + 'condition': { + 'uri': user_Ed25519.condition_uri, + 'details': { + 'type': 'ed25519-sha-256', + 'public_key': b58encode(user_Ed25519.public_key).decode(), + }, + }, + 'public_keys': [user_pub], + 'amount': '1', + } + + cond = Output(user_Ed25519, [user_pub], 1) + + assert cond.to_dict() == expected + + +def test_output_deserialization(user_Ed25519, user_pub): + from bigchaindb.common.transaction import Output + + expected = Output(user_Ed25519, [user_pub], 1) + cond = { + 'condition': { + 'uri': user_Ed25519.condition_uri, + 'details': { + 'type': 'ed25519-sha-256', + 'public_key': b58encode(user_Ed25519.public_key).decode(), + }, + }, + 'public_keys': [user_pub], + 'amount': '1', + } + cond = Output.from_dict(cond) + + assert cond == expected + + +def test_output_hashlock_serialization(): + from bigchaindb.common.transaction import Output + from cryptoconditions import PreimageSha256 + + secret = b'wow much secret' + hashlock = PreimageSha256(preimage=secret).condition_uri + + expected = { + 'condition': { + 'uri': hashlock, + }, + 'public_keys': None, + 'amount': '1', + } + cond = Output(hashlock, amount=1) + + assert cond.to_dict() == expected + + +def test_output_hashlock_deserialization(): + from bigchaindb.common.transaction import Output + from cryptoconditions import PreimageSha256 + + secret = b'wow much secret' + hashlock = PreimageSha256(preimage=secret).condition_uri + expected = Output(hashlock, amount=1) + + cond = { + 'condition': { + 'uri': hashlock + }, + 'public_keys': None, + 'amount': '1', + } + cond = Output.from_dict(cond) + + assert cond == expected + + +def test_invalid_output_initialization(cond_uri, user_pub): + from bigchaindb.common.transaction import Output + from bigchaindb.common.exceptions import AmountError + + with raises(TypeError): + Output(cond_uri, user_pub) + with raises(TypeError): + Output(cond_uri, [user_pub], 'amount') + with raises(AmountError): + Output(cond_uri, [user_pub], 0) + + +def test_generate_output_split_half_recursive(user_pub, user2_pub, user3_pub): + from bigchaindb.common.transaction import Output + from cryptoconditions import Ed25519Sha256, ThresholdSha256 + + expected_simple1 = Ed25519Sha256(public_key=b58decode(user_pub)) + expected_simple2 = Ed25519Sha256(public_key=b58decode(user2_pub)) + expected_simple3 = Ed25519Sha256(public_key=b58decode(user3_pub)) + + expected = ThresholdSha256(threshold=2) + expected.add_subfulfillment(expected_simple1) + expected_threshold = ThresholdSha256(threshold=2) + expected_threshold.add_subfulfillment(expected_simple2) + expected_threshold.add_subfulfillment(expected_simple3) + expected.add_subfulfillment(expected_threshold) + + cond = Output.generate([user_pub, [user2_pub, expected_simple3]], 1) + assert cond.fulfillment.to_dict() == expected.to_dict() + + +def test_generate_outputs_split_half_single_owner(user_pub, + user2_pub, user3_pub): + from bigchaindb.common.transaction import Output + from cryptoconditions import Ed25519Sha256, ThresholdSha256 + + expected_simple1 = Ed25519Sha256(public_key=b58decode(user_pub)) + expected_simple2 = Ed25519Sha256(public_key=b58decode(user2_pub)) + expected_simple3 = Ed25519Sha256(public_key=b58decode(user3_pub)) + + expected = ThresholdSha256(threshold=2) + expected_threshold = ThresholdSha256(threshold=2) + expected_threshold.add_subfulfillment(expected_simple2) + expected_threshold.add_subfulfillment(expected_simple3) + expected.add_subfulfillment(expected_threshold) + expected.add_subfulfillment(expected_simple1) + + cond = Output.generate([[expected_simple2, user3_pub], user_pub], 1) + assert cond.fulfillment.to_dict() == expected.to_dict() + + +def test_generate_outputs_flat_ownage(user_pub, user2_pub, user3_pub): + from bigchaindb.common.transaction import Output + from cryptoconditions import Ed25519Sha256, ThresholdSha256 + + expected_simple1 = Ed25519Sha256(public_key=b58decode(user_pub)) + expected_simple2 = Ed25519Sha256(public_key=b58decode(user2_pub)) + expected_simple3 = Ed25519Sha256(public_key=b58decode(user3_pub)) + + expected = ThresholdSha256(threshold=3) + expected.add_subfulfillment(expected_simple1) + expected.add_subfulfillment(expected_simple2) + expected.add_subfulfillment(expected_simple3) + + cond = Output.generate([user_pub, user2_pub, expected_simple3], 1) + assert cond.fulfillment.to_dict() == expected.to_dict() + + +def test_generate_output_single_owner(user_pub): + from bigchaindb.common.transaction import Output + from cryptoconditions import Ed25519Sha256 + + expected = Ed25519Sha256(public_key=b58decode(user_pub)) + cond = Output.generate([user_pub], 1) + + assert cond.fulfillment.to_dict() == expected.to_dict() + + +def test_generate_output_single_owner_with_output(user_pub): + from bigchaindb.common.transaction import Output + from cryptoconditions import Ed25519Sha256 + + expected = Ed25519Sha256(public_key=b58decode(user_pub)) + cond = Output.generate([expected], 1) + + assert cond.fulfillment.to_dict() == expected.to_dict() + + +def test_generate_output_invalid_parameters(user_pub, user2_pub, user3_pub): + from bigchaindb.common.transaction import Output + from bigchaindb.common.exceptions import AmountError + + with raises(ValueError): + Output.generate([], 1) + with raises(TypeError): + Output.generate('not a list', 1) + with raises(ValueError): + Output.generate([[user_pub, [user2_pub, [user3_pub]]]], 1) + with raises(ValueError): + Output.generate([[user_pub]], 1) + with raises(AmountError): + Output.generate([[user_pub]], -1) + + +def test_invalid_transaction_initialization(asset_definition): + from bigchaindb.common.transaction import Transaction + + with raises(ValueError): + Transaction(operation='invalid operation', asset=asset_definition) + with raises(TypeError): + Transaction(operation='CREATE', asset='invalid asset') + with raises(TypeError): + Transaction(operation='TRANSFER', asset={}) + with raises(TypeError): + Transaction( + operation='CREATE', + asset=asset_definition, + outputs='invalid outputs' + ) + with raises(TypeError): + Transaction( + operation='CREATE', + asset=asset_definition, + outputs=[], + inputs='invalid inputs' + ) + with raises(TypeError): + Transaction( + operation='CREATE', + asset=asset_definition, + outputs=[], + inputs=[], + metadata='invalid metadata' + ) + + +def test_create_default_asset_on_tx_initialization(asset_definition): + from bigchaindb.common.transaction import Transaction + + expected = {'data': None} + tx = Transaction(Transaction.CREATE, asset=expected) + asset = tx.asset + + assert asset == expected + + +def test_transaction_serialization(user_input, user_output, data): + from bigchaindb.common.transaction import Transaction + + expected = { + 'id': None, + 'version': Transaction.VERSION, + # NOTE: This test assumes that Inputs and Outputs can + # successfully be serialized + 'inputs': [user_input.to_dict()], + 'outputs': [user_output.to_dict()], + 'operation': Transaction.CREATE, + 'metadata': None, + 'asset': { + 'data': data, + } + } + + tx = Transaction(Transaction.CREATE, {'data': data}, [user_input], + [user_output]) + tx_dict = tx.to_dict() + + assert tx_dict == expected + + +def test_transaction_deserialization(tri_state_transaction): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + tx = Transaction.from_dict(tri_state_transaction) + validate_transaction_model(tx) + + +def test_invalid_input_initialization(user_input, user_pub): + from bigchaindb.common.transaction import Input + + with raises(TypeError): + Input(user_input, user_pub) + with raises(TypeError): + Input(user_input, tx_input='somethingthatiswrong') + + +def test_transaction_link_serialization(): + from bigchaindb.common.transaction import TransactionLink + + tx_id = 'a transaction id' + expected = { + 'transaction_id': tx_id, + 'output_index': 0, + } + tx_link = TransactionLink(tx_id, 0) + + assert tx_link.to_dict() == expected + + +def test_transaction_link_serialization_with_empty_payload(): + from bigchaindb.common.transaction import TransactionLink + + expected = None + tx_link = TransactionLink() + + assert tx_link.to_dict() == expected + + +def test_transaction_link_deserialization(): + from bigchaindb.common.transaction import TransactionLink + + tx_id = 'a transaction id' + expected = TransactionLink(tx_id, 0) + tx_link = { + 'transaction_id': tx_id, + 'output_index': 0, + } + tx_link = TransactionLink.from_dict(tx_link) + + assert tx_link == expected + + +def test_transaction_link_deserialization_with_empty_payload(): + from bigchaindb.common.transaction import TransactionLink + + expected = TransactionLink() + tx_link = TransactionLink.from_dict(None) + + assert tx_link == expected + + +def test_transaction_link_empty_to_uri(): + from bigchaindb.common.transaction import TransactionLink + + expected = None + tx_link = TransactionLink().to_uri() + + assert expected == tx_link + + +def test_transaction_link_to_uri(): + from bigchaindb.common.transaction import TransactionLink + + expected = 'path/transactions/abc/outputs/0' + tx_link = TransactionLink('abc', 0).to_uri('path') + + assert expected == tx_link + + +def test_cast_transaction_link_to_boolean(): + from bigchaindb.common.transaction import TransactionLink + + assert bool(TransactionLink()) is False + assert bool(TransactionLink('a', None)) is False + assert bool(TransactionLink(None, 'b')) is False + assert bool(TransactionLink('a', 'b')) is True + assert bool(TransactionLink(False, False)) is True + + +def test_transaction_link_eq(): + from bigchaindb.common.transaction import TransactionLink + + assert TransactionLink(1, 2) == TransactionLink(1, 2) + assert TransactionLink(2, 2) != TransactionLink(1, 2) + assert TransactionLink(1, 1) != TransactionLink(1, 2) + assert TransactionLink(2, 1) != TransactionLink(1, 2) + + +def test_add_input_to_tx(user_input, asset_definition): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + tx = Transaction(Transaction.CREATE, asset_definition, [], []) + tx.add_input(user_input) + + assert len(tx.inputs) == 1 + + validate_transaction_model(tx) + + +def test_add_input_to_tx_with_invalid_parameters(asset_definition): + from bigchaindb.common.transaction import Transaction + tx = Transaction(Transaction.CREATE, asset_definition) + + with raises(TypeError): + tx.add_input('somewronginput') + + +def test_add_output_to_tx(user_output, user_input, asset_definition): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + tx = Transaction(Transaction.CREATE, asset_definition, [user_input]) + tx.add_output(user_output) + + assert len(tx.outputs) == 1 + + validate_transaction_model(tx) + + +def test_add_output_to_tx_with_invalid_parameters(asset_definition): + from bigchaindb.common.transaction import Transaction + tx = Transaction(Transaction.CREATE, asset_definition, [], []) + + with raises(TypeError): + tx.add_output('somewronginput') + + +def test_sign_with_invalid_parameters(utx, user_priv): + with raises(TypeError): + utx.sign(None) + with raises(TypeError): + utx.sign(user_priv) + + +def test_validate_tx_simple_create_signature(user_input, user_output, user_priv, + asset_definition): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + tx = Transaction(Transaction.CREATE, asset_definition, [user_input], [user_output]) + expected = deepcopy(user_output) + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() + expected.fulfillment.sign(message, b58decode(user_priv)) + tx.sign([user_priv]) + + assert tx.inputs[0].to_dict()['fulfillment'] == \ + expected.fulfillment.serialize_uri() + assert tx.inputs_valid() is True + + validate_transaction_model(tx) + + +def test_invoke_simple_signature_fulfillment_with_invalid_params(utx, + user_input): + from bigchaindb.common.exceptions import KeypairMismatchException + + with raises(KeypairMismatchException): + invalid_key_pair = {'wrong_pub_key': 'wrong_priv_key'} + utx._sign_simple_signature_fulfillment(user_input, + 'somemessage', + invalid_key_pair) + + +def test_sign_threshold_with_invalid_params(utx, user_user2_threshold_input, + user3_pub, user3_priv): + from bigchaindb.common.exceptions import KeypairMismatchException + + with raises(KeypairMismatchException): + utx._sign_threshold_signature_fulfillment(user_user2_threshold_input, + 'somemessage', + {user3_pub: user3_priv}) + with raises(KeypairMismatchException): + user_user2_threshold_input.owners_before = [58 * 'a'] + utx._sign_threshold_signature_fulfillment(user_user2_threshold_input, + 'somemessage', + None) + + +def test_validate_input_with_invalid_parameters(utx): + from bigchaindb.common.transaction import Transaction + + input_conditions = [out.fulfillment.condition_uri for out in utx.outputs] + tx_dict = utx.to_dict() + tx_serialized = Transaction._to_str(tx_dict) + valid = utx._input_valid(utx.inputs[0], tx_serialized, input_conditions[0]) + assert not valid + + +def test_validate_tx_threshold_create_signature(user_user2_threshold_input, + user_user2_threshold_output, + user_pub, + user2_pub, + user_priv, + user2_priv, + asset_definition): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + tx = Transaction(Transaction.CREATE, asset_definition, + [user_user2_threshold_input], + [user_user2_threshold_output]) + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() + expected = deepcopy(user_user2_threshold_output) + expected.fulfillment.subconditions[0]['body'].sign( + message, b58decode(user_priv)) + expected.fulfillment.subconditions[1]['body'].sign( + message, b58decode(user2_priv)) + tx.sign([user_priv, user2_priv]) + + assert tx.inputs[0].to_dict()['fulfillment'] == \ + expected.fulfillment.serialize_uri() + assert tx.inputs_valid() is True + + validate_transaction_model(tx) + + +def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, + asset_definition): + from cryptoconditions import Ed25519Sha256, ThresholdSha256 + from bigchaindb.common.transaction import Input, Output, Transaction + + threshold = ThresholdSha256(threshold=2) + threshold.add_subfulfillment( + Ed25519Sha256(public_key=b58decode(user_pub))) + threshold.add_subfulfillment( + Ed25519Sha256(public_key=b58decode(user_pub))) + + threshold_input = Input(threshold, [user_pub, user_pub]) + threshold_output = Output(threshold, [user_pub, user_pub]) + + tx = Transaction(Transaction.CREATE, asset_definition, + [threshold_input], [threshold_output]) + + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() + + expected = deepcopy(threshold_input) + expected.fulfillment.subconditions[0]['body'].sign( + message, b58decode(user_priv)) + expected.fulfillment.subconditions[1]['body'].sign( + message, b58decode(user_priv)) + + tx.sign([user_priv, user_priv]) + + subconditions = tx.inputs[0].fulfillment.subconditions + expected_subconditions = expected.fulfillment.subconditions + assert subconditions[0]['body'].to_dict()['signature'] == \ + expected_subconditions[0]['body'].to_dict()['signature'] + assert subconditions[1]['body'].to_dict()['signature'] == \ + expected_subconditions[1]['body'].to_dict()['signature'] + + assert tx.inputs[0].to_dict()['fulfillment'] == \ + expected.fulfillment.serialize_uri() + assert tx.inputs_valid() is True + + +def test_multiple_input_validation_of_transfer_tx(user_input, user_output, + user_priv, user2_pub, + user2_priv, user3_pub, + user3_priv, + asset_definition): + from bigchaindb.common.transaction import (Transaction, TransactionLink, + Input, Output) + from cryptoconditions import Ed25519Sha256 + from .utils import validate_transaction_model + + tx = Transaction(Transaction.CREATE, asset_definition, [user_input], + [user_output, deepcopy(user_output)]) + tx.sign([user_priv]) + + inputs = [Input(cond.fulfillment, cond.public_keys, + TransactionLink(tx.id, index)) + for index, cond in enumerate(tx.outputs)] + outputs = [Output(Ed25519Sha256(public_key=b58decode(user3_pub)), + [user3_pub]), + Output(Ed25519Sha256(public_key=b58decode(user3_pub)), + [user3_pub])] + transfer_tx = Transaction('TRANSFER', {'id': tx.id}, inputs, outputs) + transfer_tx = transfer_tx.sign([user_priv]) + + assert transfer_tx.inputs_valid(tx.outputs) is True + + validate_transaction_model(tx) + + +def test_validate_inputs_of_transfer_tx_with_invalid_params( + transfer_tx, cond_uri, utx, user2_pub, user_priv, ffill_uri): + from bigchaindb.common.transaction import Output + from cryptoconditions import Ed25519Sha256 + + invalid_out = Output(Ed25519Sha256.from_uri(ffill_uri), ['invalid']) + assert transfer_tx.inputs_valid([invalid_out]) is False + invalid_out = utx.outputs[0] + invalid_out.public_key = 'invalid' + assert transfer_tx.inputs_valid([invalid_out]) is True + + with raises(TypeError): + assert transfer_tx.inputs_valid(None) is False + with raises(AttributeError): + transfer_tx.inputs_valid('not a list') + with raises(ValueError): + transfer_tx.inputs_valid([]) + with raises(TypeError): + transfer_tx.operation = "Operation that doesn't exist" + transfer_tx.inputs_valid([utx.outputs[0]]) + + +def test_create_create_transaction_single_io(user_output, user_pub, data): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + expected = { + 'outputs': [user_output.to_dict()], + 'metadata': data, + 'asset': { + 'data': data, + }, + 'inputs': [ + { + 'owners_before': [ + user_pub + ], + 'fulfillment': None, + 'fulfills': None + } + ], + 'operation': 'CREATE', + 'version': Transaction.VERSION, + } + + tx = Transaction.create([user_pub], [([user_pub], 1)], metadata=data, + asset=data) + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + tx_dict.pop('id') + + assert tx_dict == expected + + validate_transaction_model(tx) + + +def test_validate_single_io_create_transaction(user_pub, user_priv, data, + asset_definition): + from bigchaindb.common.transaction import Transaction + + tx = Transaction.create([user_pub], [([user_pub], 1)], metadata=data) + tx = tx.sign([user_priv]) + assert tx.inputs_valid() is True + + +def test_create_create_transaction_multiple_io(user_output, user2_output, user_pub, + user2_pub, asset_definition): + from bigchaindb.common.transaction import Transaction, Input + + # a fulfillment for a create transaction with multiple `owners_before` + # is a fulfillment for an implicit threshold condition with + # weight = len(owners_before) + input = Input.generate([user_pub, user2_pub]).to_dict() + expected = { + 'outputs': [user_output.to_dict(), user2_output.to_dict()], + 'metadata': { + 'message': 'hello' + }, + 'inputs': [input], + 'operation': 'CREATE', + 'version': Transaction.VERSION + } + tx = Transaction.create([user_pub, user2_pub], + [([user_pub], 1), ([user2_pub], 1)], + metadata={'message': 'hello'}).to_dict() + tx.pop('id') + tx.pop('asset') + + assert tx == expected + + +def test_validate_multiple_io_create_transaction(user_pub, user_priv, + user2_pub, user2_priv, + asset_definition): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + tx = Transaction.create([user_pub, user2_pub], + [([user_pub], 1), ([user2_pub], 1)], + metadata={'message': 'hello'}) + tx = tx.sign([user_priv, user2_priv]) + assert tx.inputs_valid() is True + + validate_transaction_model(tx) + + +def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, + user_user2_threshold_output, + user_user2_threshold_input, data): + from bigchaindb.common.transaction import Transaction + + expected = { + 'outputs': [user_user2_threshold_output.to_dict()], + 'metadata': data, + 'asset': { + 'data': data, + }, + 'inputs': [ + { + 'owners_before': [ + user_pub, + ], + 'fulfillment': None, + 'fulfills': None, + }, + ], + 'operation': 'CREATE', + 'version': Transaction.VERSION + } + tx = Transaction.create([user_pub], [([user_pub, user2_pub], 1)], + metadata=data, asset=data) + tx_dict = tx.to_dict() + tx_dict.pop('id') + tx_dict['inputs'][0]['fulfillment'] = None + + assert tx_dict == expected + + +def test_validate_threshold_create_transaction(user_pub, user_priv, user2_pub, + data, asset_definition): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + tx = Transaction.create([user_pub], [([user_pub, user2_pub], 1)], + metadata=data) + tx = tx.sign([user_priv]) + assert tx.inputs_valid() is True + + validate_transaction_model(tx) + + +def test_create_create_transaction_with_invalid_parameters(user_pub): + from bigchaindb.common.transaction import Transaction + + with raises(TypeError): + Transaction.create('not a list') + with raises(TypeError): + Transaction.create([], 'not a list') + with raises(ValueError): + Transaction.create([], [user_pub]) + with raises(ValueError): + Transaction.create([user_pub], []) + with raises(ValueError): + Transaction.create([user_pub], [user_pub]) + with raises(ValueError): + Transaction.create([user_pub], [([user_pub],)]) + with raises(TypeError): + Transaction.create([user_pub], [([user_pub], 1)], + metadata='not a dict or none') + with raises(TypeError): + Transaction.create([user_pub], + [([user_pub], 1)], + asset='not a dict or none') + + +def test_outputs_to_inputs(tx): + inputs = tx.to_inputs([0]) + assert len(inputs) == 1 + input = inputs.pop() + assert input.owners_before == tx.outputs[0].public_keys + assert input.fulfillment == tx.outputs[0].fulfillment + assert input.fulfills.txid == tx.id + assert input.fulfills.output == 0 + + +def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, + user2_output, user_priv): + from bigchaindb.common.transaction import Transaction + from .utils import validate_transaction_model + + expected = { + 'id': None, + 'outputs': [user2_output.to_dict()], + 'metadata': None, + 'asset': { + 'id': tx.id, + }, + 'inputs': [ + { + 'owners_before': [ + user_pub + ], + 'fulfillment': None, + 'fulfills': { + 'transaction_id': tx.id, + 'output_index': 0 + } + } + ], + 'operation': 'TRANSFER', + 'version': Transaction.VERSION + } + inputs = tx.to_inputs([0]) + transfer_tx = Transaction.transfer(inputs, [([user2_pub], 1)], + asset_id=tx.id) + transfer_tx = transfer_tx.sign([user_priv]) + transfer_tx = transfer_tx.to_dict() + + expected_input = deepcopy(inputs[0]) + json_serialized_tx = json.dumps(expected, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(json_serialized_tx.encode()) + message.update('{}{}'.format( + expected['inputs'][0]['fulfills']['transaction_id'], + expected['inputs'][0]['fulfills']['output_index'], + ).encode()) + expected_input.fulfillment.sign(message.digest(), b58decode(user_priv)) + expected_ffill = expected_input.fulfillment.serialize_uri() + transfer_ffill = transfer_tx['inputs'][0]['fulfillment'] + + assert transfer_ffill == expected_ffill + + transfer_tx = Transaction.from_dict(transfer_tx) + assert transfer_tx.inputs_valid([tx.outputs[0]]) is True + + validate_transaction_model(transfer_tx) + + +def test_create_transfer_transaction_multiple_io(user_pub, user_priv, + user2_pub, user2_priv, + user3_pub, user2_output, + asset_definition): + from bigchaindb.common.transaction import Transaction + + tx = Transaction.create([user_pub], [([user_pub], 1), ([user2_pub], 1)], + metadata={'message': 'hello'}) + tx = tx.sign([user_priv]) + + expected = { + 'outputs': [user2_output.to_dict(), user2_output.to_dict()], + 'metadata': None, + 'inputs': [ + { + 'owners_before': [ + user_pub + ], + 'fulfillment': None, + 'fulfills': { + 'transaction_id': tx.id, + 'output_index': 0 + } + }, { + 'owners_before': [ + user2_pub + ], + 'fulfillment': None, + 'fulfills': { + 'transaction_id': tx.id, + 'output_index': 1 + } + } + ], + 'operation': 'TRANSFER', + 'version': Transaction.VERSION + } + + transfer_tx = Transaction.transfer(tx.to_inputs(), + [([user2_pub], 1), ([user2_pub], 1)], + asset_id=tx.id) + transfer_tx = transfer_tx.sign([user_priv, user2_priv]) + + assert len(transfer_tx.inputs) == 2 + assert len(transfer_tx.outputs) == 2 + + assert transfer_tx.inputs_valid(tx.outputs) is True + + transfer_tx = transfer_tx.to_dict() + transfer_tx['inputs'][0]['fulfillment'] = None + transfer_tx['inputs'][1]['fulfillment'] = None + transfer_tx.pop('asset') + transfer_tx.pop('id') + + assert expected == transfer_tx + + +def test_create_transfer_with_invalid_parameters(tx, user_pub): + from bigchaindb.common.transaction import Transaction + + with raises(TypeError): + Transaction.transfer({}, [], tx.id) + with raises(ValueError): + Transaction.transfer([], [], tx.id) + with raises(TypeError): + Transaction.transfer(['fulfillment'], {}, tx.id) + with raises(ValueError): + Transaction.transfer(['fulfillment'], [], tx.id) + with raises(ValueError): + Transaction.transfer(['fulfillment'], [user_pub], tx.id) + with raises(ValueError): + Transaction.transfer(['fulfillment'], [([user_pub],)], tx.id) + with raises(TypeError): + Transaction.transfer(['fulfillment'], [([user_pub], 1)], + tx.id, metadata='not a dict or none') + with raises(TypeError): + Transaction.transfer(['fulfillment'], [([user_pub], 1)], + ['not a string']) + + +def test_cant_add_empty_output(): + from bigchaindb.common.transaction import Transaction + tx = Transaction(Transaction.CREATE, None) + + with raises(TypeError): + tx.add_output(None) + + +def test_cant_add_empty_input(): + from bigchaindb.common.transaction import Transaction + tx = Transaction(Transaction.CREATE, None) + + with raises(TypeError): + tx.add_input(None) + + +def test_unfulfilled_transaction_serialized(unfulfilled_transaction): + from bigchaindb.common.transaction import Transaction + tx_obj = Transaction.from_dict(unfulfilled_transaction) + expected = json.dumps(unfulfilled_transaction, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + assert tx_obj.serialized == expected + + +def test_fulfilled_transaction_serialized(fulfilled_transaction): + from bigchaindb.common.transaction import Transaction + tx_obj = Transaction.from_dict(fulfilled_transaction) + expected = json.dumps(fulfilled_transaction, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + assert tx_obj.serialized == expected + + +def test_transaction_hash(fulfilled_transaction): + from bigchaindb.common.transaction import Transaction + tx_obj = Transaction.from_dict(fulfilled_transaction) + assert tx_obj._id is None + assert tx_obj.id is None + thing_to_hash = json.dumps(fulfilled_transaction, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + expected_hash_id = sha3_256(thing_to_hash.encode()).hexdigest() + tx_obj._hash() + assert tx_obj._id == expected_hash_id + assert tx_obj.id == expected_hash_id + + +def test_output_from_dict_invalid_amount(user_output): + from bigchaindb.common.transaction import Output + from bigchaindb.common.exceptions import AmountError + + out = user_output.to_dict() + out['amount'] = 'a' + with raises(AmountError): + Output.from_dict(out) + + +def test_unspent_outputs_property(merlin, alice, bob, carol): + from bigchaindb.common.transaction import Transaction + tx = Transaction.create( + [merlin.public_key], + [([alice.public_key], 1), + ([bob.public_key], 2), + ([carol.public_key], 3)], + asset={'hash': '06e47bcf9084f7ecfd2a2a2ad275444a'}, + ).sign([merlin.private_key]) + unspent_outputs = list(tx.unspent_outputs) + assert len(unspent_outputs) == 3 + assert all(utxo.transaction_id == tx.id for utxo in unspent_outputs) + assert all(utxo.asset_id == tx.id for utxo in unspent_outputs) + assert all( + utxo.output_index == i for i, utxo in enumerate(unspent_outputs)) + unspent_output_0 = unspent_outputs[0] + assert unspent_output_0.amount == 1 + assert unspent_output_0.condition_uri == Ed25519Sha256( + public_key=b58decode(alice.public_key)).condition_uri + unspent_output_1 = unspent_outputs[1] + assert unspent_output_1.amount == 2 + assert unspent_output_1.condition_uri == Ed25519Sha256( + public_key=b58decode(bob.public_key)).condition_uri + unspent_output_2 = unspent_outputs[2] + assert unspent_output_2.amount == 3 + assert unspent_output_2.condition_uri == Ed25519Sha256( + public_key=b58decode(carol.public_key)).condition_uri + + +def test_spent_outputs_property(signed_transfer_tx): + spent_outputs = list(signed_transfer_tx.spent_outputs) + tx = signed_transfer_tx.to_dict() + assert len(spent_outputs) == 1 + spent_output = spent_outputs[0] + assert spent_output['transaction_id'] == tx['inputs'][0]['fulfills']['transaction_id'] + assert spent_output['output_index'] == tx['inputs'][0]['fulfills']['output_index'] + # assert spent_output._asdict() == tx['inputs'][0]['fulfills'] diff --git a/tests/common/utils.py b/tests/common/utils.py new file mode 100644 index 0000000..430ed73 --- /dev/null +++ b/tests/common/utils.py @@ -0,0 +1,15 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +def validate_transaction_model(tx): + from bigchaindb.common.transaction import Transaction + from bigchaindb.common.schema import validate_transaction_schema + + tx_dict = tx.to_dict() + # Check that a transaction is valid by re-serializing it + # And calling validate_transaction_schema + validate_transaction_schema(tx_dict) + Transaction.from_dict(tx_dict) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..64a4311 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,768 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Fixtures and setup / teardown functions + +Tasks: +1. setup test database before starting the tests +2. delete test database after running the tests +""" +import json +import os +import copy +import random +import tempfile +from collections import namedtuple +from logging import getLogger +from logging.config import dictConfig + +import pytest +from pymongo import MongoClient + +from bigchaindb import ValidatorElection +from bigchaindb.common import crypto +from bigchaindb.common.transaction_mode_types import BROADCAST_TX_COMMIT +from bigchaindb.tendermint_utils import key_from_base64 +from bigchaindb.backend import schema, query +from bigchaindb.common.crypto import (key_pair_from_ed25519_key, + public_key_from_ed25519_key) +from bigchaindb.common.exceptions import DatabaseDoesNotExist +from bigchaindb.lib import Block +from tests.utils import gen_vote + +TEST_DB_NAME = 'bigchain_test' + +USER2_SK, USER2_PK = crypto.generate_key_pair() + +# Test user. inputs will be created for this user. Cryptography Keys +USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie' +USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' + + +def pytest_addoption(parser): + from bigchaindb.backend.connection import BACKENDS + + backends = ', '.join(BACKENDS.keys()) + parser.addoption( + '--database-backend', + action='store', + default=os.environ.get('PLANETMINT_DATABASE_BACKEND', 'localmongodb'), + help='Defines the backend to use (available: {})'.format(backends), + ) + + +def pytest_configure(config): + config.addinivalue_line( + 'markers', + 'bdb(): Mark the test as needing Planetmint.' + 'Planetmint will be configured such that the database and tables are available for an ' + 'entire test session.' + 'You need to run a backend (e.g. MongoDB) ' + 'prior to running tests with this marker. You should not need to restart the backend ' + 'in between tests runs since the test infrastructure flushes the backend upon session end.' + ) + config.addinivalue_line( + 'markers', + 'abci(): Mark the test as needing a running ABCI server in place. Use this marker' + 'for tests that require a running Tendermint instance. Note that the test infrastructure' + 'has no way to reset Tendermint data upon session end - you need to do it manually.' + 'Setup performed by this marker includes the steps performed by the bdb marker.' + ) + + +@pytest.fixture(autouse=True) +def _bdb_marker(request): + if request.keywords.get('bdb', None): + request.getfixturevalue('_bdb') + + +@pytest.fixture(autouse=True) +def _restore_config(_configure_bigchaindb): + from bigchaindb import config, config_utils + config_before_test = copy.deepcopy(config) + yield + config_utils.set_config(config_before_test) + + +@pytest.fixture(scope='session') +def _configure_bigchaindb(request): + import bigchaindb + from bigchaindb import config_utils + test_db_name = TEST_DB_NAME + # Put a suffix like _gw0, _gw1 etc on xdist processes + xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid') + if xdist_suffix: + test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) + + backend = request.config.getoption('--database-backend') + + config = { + 'database': bigchaindb._database_map[backend], + 'tendermint': { + 'host': 'localhost', + 'port': 26657, + } + } + config['database']['name'] = test_db_name + config = config_utils.env_config(config) + config_utils.set_config(config) + + +@pytest.fixture(scope='session') +def _setup_database(_configure_bigchaindb): + from bigchaindb import config + from bigchaindb.backend import connect + print('Initializing test db') + dbname = config['database']['name'] + conn = connect() + + _drop_db(conn, dbname) + schema.init_database(conn) + print('Finishing init database') + + yield + + print('Deleting `{}` database'.format(dbname)) + conn = connect() + _drop_db(conn, dbname) + + print('Finished deleting `{}`'.format(dbname)) + + +@pytest.fixture +def _bdb(_setup_database, _configure_bigchaindb): + from bigchaindb import config + from bigchaindb.backend import connect + from .utils import flush_db + from bigchaindb.common.memoize import to_dict, from_dict + from bigchaindb.models import Transaction + conn = connect() + yield + dbname = config['database']['name'] + flush_db(conn, dbname) + + to_dict.cache_clear() + from_dict.cache_clear() + Transaction._input_valid.cache_clear() + + +# We need this function to avoid loading an existing +# conf file located in the home of the user running +# the tests. If it's too aggressive we can change it +# later. +@pytest.fixture +def ignore_local_config_file(monkeypatch): + def mock_file_config(filename=None): + return {} + + monkeypatch.setattr('bigchaindb.config_utils.file_config', + mock_file_config) + + +@pytest.fixture +def reset_logging_config(): + # root_logger_level = getLogger().level + root_logger_level = 'DEBUG' + dictConfig({'version': 1, 'root': {'level': 'NOTSET'}}) + yield + getLogger().setLevel(root_logger_level) + + +@pytest.fixture +def user_sk(): + return USER_PRIVATE_KEY + + +@pytest.fixture +def user_pk(): + return USER_PUBLIC_KEY + + +@pytest.fixture +def user2_sk(): + return USER2_SK + + +@pytest.fixture +def user2_pk(): + return USER2_PK + + +@pytest.fixture +def alice(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def bob(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def bob_privkey(bob): + return bob.private_key + + +@pytest.fixture +def bob_pubkey(carol): + return bob.public_key + + +@pytest.fixture +def carol(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def carol_privkey(carol): + return carol.private_key + + +@pytest.fixture +def carol_pubkey(carol): + return carol.public_key + + +@pytest.fixture +def merlin(): + from bigchaindb.common.crypto import generate_key_pair + return generate_key_pair() + + +@pytest.fixture +def a(): + from abci import types_v0_31_5 + return types_v0_31_5 + + +@pytest.fixture +def b(): + from bigchaindb import BigchainDB + return BigchainDB) + + +@pytest.fixture +def b_mock(b, network_validators): + b.get_validators = mock_get_validators(network_validators) + + return b + + +def mock_get_validators(network_validators): + def validator_set(height): + validators = [] + for public_key, power in network_validators.items(): + validators.append({ + 'public_key': {'type': 'ed25519-base64', 'value': public_key}, + 'voting_power': power + }) + return validators + + return validator_set + + +@pytest.fixture +def create_tx(alice, user_pk): + from bigchaindb.models import Transaction + name = f'I am created by the create_tx fixture. My random identifier is {random.random()}.' + return Transaction.create([alice.public_key], [([user_pk], 1)], asset={'name': name}) + + +@pytest.fixture +def signed_create_tx(alice, create_tx): + return create_tx.sign([alice.private_key]) + + +@pytest.fixture +def posted_create_tx(b, signed_create_tx): + res = b.post_transaction(signed_create_tx, BROADCAST_TX_COMMIT) + assert res.status_code == 200 + return signed_create_tx + + +@pytest.fixture +def signed_transfer_tx(signed_create_tx, user_pk, user_sk): + from bigchaindb.models import Transaction + inputs = signed_create_tx.to_inputs() + tx = Transaction.transfer(inputs, [([user_pk], 1)], asset_id=signed_create_tx.id) + return tx.sign([user_sk]) + + +@pytest.fixture +def double_spend_tx(signed_create_tx, carol_pubkey, user_sk): + from bigchaindb.models import Transaction + inputs = signed_create_tx.to_inputs() + tx = Transaction.transfer( + inputs, [([carol_pubkey], 1)], asset_id=signed_create_tx.id) + return tx.sign([user_sk]) + + +def _get_height(b): + maybe_block = b.get_latest_block() + return 0 if maybe_block is None else maybe_block['height'] + + +@pytest.fixture +def inputs(user_pk, b, alice): + from bigchaindb.models import Transaction + # create blocks with transactions for `USER` to spend + for height in range(1, 4): + transactions = [ + Transaction.create( + [alice.public_key], + [([user_pk], 1)], + metadata={'msg': random.random()}, + ).sign([alice.private_key]) + for _ in range(10) + ] + tx_ids = [tx.id for tx in transactions] + block = Block(app_hash='hash'+str(height), height=height, transactions=tx_ids) + b.store_block(block._asdict()) + b.store_bulk_transactions(transactions) + + +@pytest.fixture +def dummy_db(request): + from bigchaindb.backend import connect + + conn = connect() + dbname = request.fixturename + xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid') + if xdist_suffix: + dbname = '{}_{}'.format(dbname, xdist_suffix) + + _drop_db(conn, dbname) # make sure we start with a clean DB + schema.init_database(conn, dbname) + yield dbname + + _drop_db(conn, dbname) + + +def _drop_db(conn, dbname): + try: + schema.drop_database(conn, dbname) + except DatabaseDoesNotExist: + pass + + +@pytest.fixture +def db_config(): + from bigchaindb import config + return config['database'] + + +@pytest.fixture +def db_host(db_config): + return db_config['host'] + + +@pytest.fixture +def db_port(db_config): + return db_config['port'] + + +@pytest.fixture +def db_name(db_config): + return db_config['name'] + + +@pytest.fixture +def db_conn(): + from bigchaindb.backend import connect + return connect() + + +@pytest.fixture +def db_context(db_config, db_host, db_port, db_name, db_conn): + DBContext = namedtuple( + 'DBContext', ('config', 'host', 'port', 'name', 'conn')) + return DBContext( + config=db_config, + host=db_host, + port=db_port, + name=db_name, + conn=db_conn, + ) + + +@pytest.fixture +def tendermint_host(): + return os.getenv('PLANETMINT_TENDERMINT_HOST', 'localhost') + + +@pytest.fixture +def tendermint_port(): + return int(os.getenv('PLANETMINT_TENDERMINT_PORT', 26657)) + + +@pytest.fixture +def tendermint_ws_url(tendermint_host, tendermint_port): + return 'ws://{}:{}/websocket'.format(tendermint_host, tendermint_port) + + +@pytest.fixture(autouse=True) +def _abci_http(request): + if request.keywords.get('abci', None): + request.getfixturevalue('abci_http') + + +@pytest.fixture +def abci_http(_setup_database, _configure_bigchaindb, abci_server, + tendermint_host, tendermint_port): + import requests + import time + + for i in range(300): + try: + uri = 'http://{}:{}/abci_info'.format(tendermint_host, tendermint_port) + requests.get(uri) + return True + + except requests.exceptions.RequestException: + pass + time.sleep(1) + + return False + + +@pytest.yield_fixture(scope='session') +def event_loop(): + import asyncio + + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + +@pytest.fixture(scope='session') +def abci_server(): + from abci.server import ABCIServer + from abci import types_v0_31_5 + from bigchaindb.core import App + from bigchaindb.utils import Process + + app = ABCIServer(app=App(types_v0_31_5)) + abci_proxy = Process(name='ABCI', target=app.run) + yield abci_proxy.start() + abci_proxy.terminate() + + +@pytest.fixture +def wsserver_config(): + from bigchaindb import config + return config['wsserver'] + + +@pytest.fixture +def wsserver_scheme(wsserver_config): + return wsserver_config['advertised_scheme'] + + +@pytest.fixture +def wsserver_host(wsserver_config): + return wsserver_config['advertised_host'] + + +@pytest.fixture +def wsserver_port(wsserver_config): + return wsserver_config['advertised_port'] + + +@pytest.fixture +def wsserver_base_url(wsserver_scheme, wsserver_host, wsserver_port): + return '{}://{}:{}'.format(wsserver_scheme, wsserver_host, wsserver_port) + + +@pytest.fixture +def unspent_output_0(): + return { + 'amount': 1, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 0, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d' + } + + +@pytest.fixture +def unspent_output_1(): + return { + 'amount': 2, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 1, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + } + + +@pytest.fixture +def unspent_output_2(): + return { + 'amount': 3, + 'asset_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + 'condition_uri': 'ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072', + 'fulfillment_message': '{"asset":{"data":{"hash":"06e47bcf9084f7ecfd2a2a2ad275444a"}},"id":"e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d","inputs":[{"fulfillment":"pGSAIIQT0Jm6LDlcSs9coJK4Q4W-SNtsO2EtMtQJ04EUjBMJgUAXKIqeaippbF-IClhhZNNaP6EIZ_OgrVQYU4mH6b-Vc3Tg-k6p-rJOlLGUUo_w8C5QgPHNRYFOqUk2f1q0Cs4G","fulfills":null,"owners_before":["9taLkHkaBXeSF8vrhDGFTAmcZuCEPqjQrKadfYGs4gHv"]}],"metadata":null,"operation":"CREATE","outputs":[{"amount":"1","condition":{"details":{"public_key":"6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz","type":"ed25519-sha-256"},"uri":"ni:///sha-256;RmovleG60-7K0CX60jjfUunV3lBpUOkiQOAnBzghm0w?fpt=ed25519-sha-256&cost=131072"},"public_keys":["6FDGsHrR9RZqNaEm7kBvqtxRkrvuWogBW2Uy7BkWc5Tz"]},{"amount":"2","condition":{"details":{"public_key":"AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT","type":"ed25519-sha-256"},"uri":"ni:///sha-256;-HlYmgwwl-vXwE52IaADhvYxaL1TbjqfJ-LGn5a1PFc?fpt=ed25519-sha-256&cost=131072"},"public_keys":["AH9D7xgmhyLmVE944zvHvuvYWuj5DfbMBJhnDM4A5FdT"]},{"amount":"3","condition":{"details":{"public_key":"HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB","type":"ed25519-sha-256"},"uri":"ni:///sha-256;xfn8pvQkTCPtvR0trpHy2pqkkNTmMBCjWMMOHtk3WO4?fpt=ed25519-sha-256&cost=131072"},"public_keys":["HpmSVrojHvfCXQbmoAs4v6Aq1oZiZsZDnjr68KiVtPbB"]}],"version":"1.0"}', # noqa + 'output_index': 2, + 'transaction_id': 'e897c7a0426461a02b4fca8ed73bc0debed7570cf3b40fb4f49c963434225a4d', + } + + +@pytest.fixture +def unspent_outputs(unspent_output_0, unspent_output_1, unspent_output_2): + return unspent_output_0, unspent_output_1, unspent_output_2 + + +@pytest.fixture +def mongo_client(db_context): + return MongoClient(host=db_context.host, port=db_context.port) + + +@pytest.fixture +def utxo_collection(db_context, mongo_client): + return mongo_client[db_context.name].utxos + + +@pytest.fixture +def dummy_unspent_outputs(): + return [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 1}, + {'transaction_id': 'b', 'output_index': 0}, + ] + + +@pytest.fixture +def utxoset(dummy_unspent_outputs, utxo_collection): + res = utxo_collection.insert_many(copy.deepcopy(dummy_unspent_outputs)) + assert res.acknowledged + assert len(res.inserted_ids) == 3 + return dummy_unspent_outputs, utxo_collection + + +@pytest.fixture +def network_validators(node_keys): + validator_pub_power = {} + voting_power = [8, 10, 7, 9] + for pub, priv in node_keys.items(): + validator_pub_power[pub] = voting_power.pop() + + return validator_pub_power + + +@pytest.fixture +def network_validators58(network_validators): + network_validators_base58 = {} + for p, v in network_validators.items(): + p = public_key_from_ed25519_key(key_from_base64(p)) + network_validators_base58[p] = v + + return network_validators_base58 + + +@pytest.fixture +def node_key(node_keys): + (pub, priv) = list(node_keys.items())[0] + return key_pair_from_ed25519_key(key_from_base64(priv)) + + +@pytest.fixture +def ed25519_node_keys(node_keys): + (pub, priv) = list(node_keys.items())[0] + node_keys_dict = {} + for pub, priv in node_keys.items(): + key = key_pair_from_ed25519_key(key_from_base64(priv)) + node_keys_dict[key.public_key] = key + + return node_keys_dict + + +@pytest.fixture +def node_keys(): + return {'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=': + 'cM5oW4J0zmUSZ/+QRoRlincvgCwR0pEjFoY//ZnnjD3Mv8Nqy8q6VdnOFI0XDHhwtFcqRIz0Y8rtjSdngUTKUw==', + 'GIijU7GBcVyiVUcB0GwWZbxCxdk2xV6pxdvL24s/AqM=': + 'mdz7IjP6mGXs6+ebgGJkn7kTXByUeeGhV+9aVthLuEAYiKNTsYFxXKJVRwHQbBZlvELF2TbFXqnF28vbiz8Cow==', + 'JbfwrLvCVIwOPm8tj8936ki7IYbmGHjPiKb6nAZegRA=': + '83VINXdj2ynOHuhvSZz5tGuOE5oYzIi0mEximkX1KYMlt/Csu8JUjA4+by2Pz3fqSLshhuYYeM+IpvqcBl6BEA==', + 'PecJ58SaNRsWJZodDmqjpCWqG6btdwXFHLyE40RYlYM=': + 'uz8bYgoL4rHErWT1gjjrnA+W7bgD/uDQWSRKDmC8otc95wnnxJo1GxYlmh0OaqOkJaobpu13BcUcvITjRFiVgw=='} + + +@pytest.fixture +def priv_validator_path(node_keys): + (public_key, private_key) = list(node_keys.items())[0] + priv_validator = { + 'address': '84F787D95E196DC5DE5F972666CFECCA36801426', + 'pub_key': { + 'type': 'AC26791624DE60', + 'value': public_key + }, + 'last_height': 0, + 'last_round': 0, + 'last_step': 0, + 'priv_key': { + 'type': '954568A3288910', + 'value': private_key + } + } + fd, path = tempfile.mkstemp() + socket = os.fdopen(fd, 'w') + json.dump(priv_validator, socket) + socket.close() + return path + + +@pytest.fixture +def bad_validator_path(node_keys): + (public_key, private_key) = list(node_keys.items())[1] + priv_validator = { + 'address': '84F787D95E196DC5DE5F972666CFECCA36801426', + 'pub_key': { + 'type': 'AC26791624DE60', + 'value': public_key + }, + 'last_height': 0, + 'last_round': 0, + 'last_step': 0, + 'priv_key': { + 'type': '954568A3288910', + 'value': private_key + } + } + fd, path = tempfile.mkstemp() + socket = os.fdopen(fd, 'w') + json.dump(priv_validator, socket) + socket.close() + return path + + +@pytest.fixture +def validators(b, node_keys): + from bigchaindb.backend import query + import time + + def timestamp(): # we need this to force unique election_ids for setup and teardown of fixtures + return str(time.time()) + + height = get_block_height(b) + + original_validators = b.get_validators() + + (public_key, private_key) = list(node_keys.items())[0] + + validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2', + 'public_key': {'value': public_key, + 'type': 'ed25519-base64'}, + 'voting_power': 10}] + + validator_update = {'validators': validator_set, + 'height': height + 1, + 'election_id': f'setup_at_{timestamp()}'} + + query.store_validator_set(b.connection, validator_update) + + yield + + height = get_block_height(b) + + validator_update = {'validators': original_validators, + 'height': height, + 'election_id': f'teardown_at_{timestamp()}'} + + query.store_validator_set(b.connection, validator_update) + + +def get_block_height(b): + + if b.get_latest_block(): + height = b.get_latest_block()['height'] + else: + height = 0 + + return height + + +@pytest.fixture +def new_validator(): + public_key = '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034' + power = 1 + node_id = 'fake_node_id' + + return {'public_key': {'value': public_key, + 'type': 'ed25519-base16'}, + 'power': power, + 'node_id': node_id} + + +@pytest.fixture +def valid_upsert_validator_election(b_mock, node_key, new_validator): + voters = ValidatorElection.recipients(b_mock) + return ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +@pytest.fixture +def valid_upsert_validator_election_2(b_mock, node_key, new_validator): + voters = ValidatorElection.recipients(b_mock) + return ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +@pytest.fixture +def ongoing_validator_election(b, valid_upsert_validator_election, ed25519_node_keys): + validators = b.get_validators(height=1) + genesis_validators = {'validators': validators, + 'height': 0} + query.store_validator_set(b.connection, genesis_validators) + b.store_bulk_transactions([valid_upsert_validator_election]) + query.store_election(b.connection, valid_upsert_validator_election.id, 1, + is_concluded=False) + block_1 = Block(app_hash='hash_1', height=1, + transactions=[valid_upsert_validator_election.id]) + b.store_block(block_1._asdict()) + return valid_upsert_validator_election + + +@pytest.fixture +def ongoing_validator_election_2(b, valid_upsert_validator_election_2, ed25519_node_keys): + validators = b.get_validators(height=1) + genesis_validators = {'validators': validators, + 'height': 0, + 'election_id': None} + query.store_validator_set(b.connection, genesis_validators) + + b.store_bulk_transactions([valid_upsert_validator_election_2]) + block_1 = Block(app_hash='hash_2', height=1, transactions=[valid_upsert_validator_election_2.id]) + b.store_block(block_1._asdict()) + return valid_upsert_validator_election_2 + + +@pytest.fixture +def validator_election_votes(b_mock, ongoing_validator_election, ed25519_node_keys): + voters = ValidatorElection.recipients(b_mock) + votes = generate_votes(ongoing_validator_election, voters, ed25519_node_keys) + return votes + + +@pytest.fixture +def validator_election_votes_2(b_mock, ongoing_validator_election_2, ed25519_node_keys): + voters = ValidatorElection.recipients(b_mock) + votes = generate_votes(ongoing_validator_election_2, voters, ed25519_node_keys) + return votes + + +def generate_votes(election, voters, keys): + votes = [] + for voter, _ in enumerate(voters): + v = gen_vote(election, voter, keys) + votes.append(v) + return votes diff --git a/tests/db/__init__.py b/tests/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py new file mode 100644 index 0000000..b5564c7 --- /dev/null +++ b/tests/db/test_bigchain_api.py @@ -0,0 +1,533 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from unittest.mock import patch + +import pytest +from base58 import b58decode + +pytestmark = pytest.mark.bdb + + +class TestBigchainApi(object): + + def test_get_spent_with_double_spend_detected(self, b, alice): + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import DoubleSpend + from bigchaindb.exceptions import CriticalDoubleSpend + + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) + tx = tx.sign([alice.private_key]) + + b.store_bulk_transactions([tx]) + + transfer_tx = Transaction.transfer(tx.to_inputs(), [([alice.public_key], 1)], + asset_id=tx.id) + transfer_tx = transfer_tx.sign([alice.private_key]) + transfer_tx2 = Transaction.transfer(tx.to_inputs(), [([alice.public_key], 2)], + asset_id=tx.id) + transfer_tx2 = transfer_tx2.sign([alice.private_key]) + + with pytest.raises(DoubleSpend): + b.validate_transaction(transfer_tx2, [transfer_tx]) + + b.store_bulk_transactions([transfer_tx]) + + with pytest.raises(DoubleSpend): + b.validate_transaction(transfer_tx2) + + b.store_bulk_transactions([transfer_tx2]) + + with pytest.raises(CriticalDoubleSpend): + b.get_spent(tx.id, 0) + + def test_double_inclusion(self, b, alice): + from bigchaindb.models import Transaction + from bigchaindb.backend.exceptions import OperationError + + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)]) + tx = tx.sign([alice.private_key]) + + b.store_bulk_transactions([tx]) + + with pytest.raises(OperationError): + b.store_bulk_transactions([tx]) + + def test_text_search(self, b, alice): + from bigchaindb.models import Transaction + + # define the assets + asset1 = {'msg': 'Planetmint 1'} + asset2 = {'msg': 'Planetmint 2'} + asset3 = {'msg': 'Planetmint 3'} + + # create the transactions + tx1 = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset1).sign([alice.private_key]) + tx2 = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset2).sign([alice.private_key]) + tx3 = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset3).sign([alice.private_key]) + + # write the transactions to the DB + b.store_bulk_transactions([tx1, tx2, tx3]) + + # get the assets through text search + assets = list(b.text_search('bigchaindb')) + assert len(assets) == 3 + + @pytest.mark.usefixtures('inputs') + def test_non_create_input_not_found(self, b, user_pk): + from cryptoconditions import Ed25519Sha256 + from bigchaindb.common.exceptions import InputDoesNotExist + from bigchaindb.common.transaction import Input, TransactionLink + from bigchaindb.models import Transaction + + # Create an input for a non existing transaction + input = Input(Ed25519Sha256(public_key=b58decode(user_pk)), + [user_pk], + TransactionLink('somethingsomething', 0)) + tx = Transaction.transfer([input], [([user_pk], 1)], + asset_id='mock_asset_link') + with pytest.raises(InputDoesNotExist): + tx.validate(b) + + def test_write_transaction(self, b, user_sk, user_pk, alice, create_tx): + from bigchaindb.models import Transaction + + asset1 = {'msg': 'Planetmint 1'} + + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset1).sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + tx_from_db = b.get_transaction(tx.id) + + before = tx.to_dict() + after = tx_from_db.to_dict() + + assert before['asset']['data'] == after['asset']['data'] + before.pop('asset', None) + after.pop('asset', None) + assert before == after + + +class TestTransactionValidation(object): + + def test_non_create_input_not_found(self, b, signed_transfer_tx): + from bigchaindb.common.exceptions import InputDoesNotExist + from bigchaindb.common.transaction import TransactionLink + + signed_transfer_tx.inputs[0].fulfills = TransactionLink('c', 0) + with pytest.raises(InputDoesNotExist): + b.validate_transaction(signed_transfer_tx) + + @pytest.mark.usefixtures('inputs') + def test_non_create_valid_input_wrong_owner(self, b, user_pk): + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.common.exceptions import InvalidSignature + from bigchaindb.models import Transaction + + input_tx = b.fastquery.get_outputs_by_public_key(user_pk).pop() + input_transaction = b.get_transaction(input_tx.txid) + sk, pk = generate_key_pair() + tx = Transaction.create([pk], [([user_pk], 1)]) + tx.operation = 'TRANSFER' + tx.asset = {'id': input_transaction.id} + tx.inputs[0].fulfills = input_tx + + with pytest.raises(InvalidSignature): + b.validate_transaction(tx) + + @pytest.mark.usefixtures('inputs') + def test_non_create_double_spend(self, b, signed_create_tx, + signed_transfer_tx, double_spend_tx): + from bigchaindb.common.exceptions import DoubleSpend + + b.store_bulk_transactions([signed_create_tx, signed_transfer_tx]) + + with pytest.raises(DoubleSpend): + b.validate_transaction(double_spend_tx) + + +class TestMultipleInputs(object): + + def test_transfer_single_owner_single_input(self, b, inputs, user_pk, + user_sk): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + + tx_link = b.fastquery.get_outputs_by_public_key(user_pk).pop() + input_tx = b.get_transaction(tx_link.txid) + inputs = input_tx.to_inputs() + tx = Transaction.transfer(inputs, [([user2_pk], 1)], + asset_id=input_tx.id) + tx = tx.sign([user_sk]) + + # validate transaction + tx.validate(b) + assert len(tx.inputs) == 1 + assert len(tx.outputs) == 1 + + def test_single_owner_before_multiple_owners_after_single_input(self, b, + user_sk, + user_pk, + inputs): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + user3_sk, user3_pk = crypto.generate_key_pair() + tx_link = b.fastquery.get_outputs_by_public_key(user_pk).pop() + + input_tx = b.get_transaction(tx_link.txid) + tx = Transaction.transfer(input_tx.to_inputs(), + [([user2_pk, user3_pk], 1)], + asset_id=input_tx.id) + tx = tx.sign([user_sk]) + + tx.validate(b) + assert len(tx.inputs) == 1 + assert len(tx.outputs) == 1 + + @pytest.mark.usefixtures('inputs') + def test_multiple_owners_before_single_owner_after_single_input(self, b, + user_sk, + user_pk, + alice): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + user3_sk, user3_pk = crypto.generate_key_pair() + + tx = Transaction.create([alice.public_key], [([user_pk, user2_pk], 1)]) + tx = tx.sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + owned_input = b.fastquery.get_outputs_by_public_key(user_pk).pop() + input_tx = b.get_transaction(owned_input.txid) + inputs = input_tx.to_inputs() + + transfer_tx = Transaction.transfer(inputs, [([user3_pk], 1)], + asset_id=input_tx.id) + transfer_tx = transfer_tx.sign([user_sk, user2_sk]) + + # validate transaction + transfer_tx.validate(b) + assert len(transfer_tx.inputs) == 1 + assert len(transfer_tx.outputs) == 1 + + @pytest.mark.usefixtures('inputs') + def test_multiple_owners_before_multiple_owners_after_single_input(self, b, + user_sk, + user_pk, + alice): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + user3_sk, user3_pk = crypto.generate_key_pair() + user4_sk, user4_pk = crypto.generate_key_pair() + + tx = Transaction.create([alice.public_key], [([user_pk, user2_pk], 1)]) + tx = tx.sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + # get input + tx_link = b.fastquery.get_outputs_by_public_key(user_pk).pop() + tx_input = b.get_transaction(tx_link.txid) + + tx = Transaction.transfer(tx_input.to_inputs(), + [([user3_pk, user4_pk], 1)], + asset_id=tx_input.id) + tx = tx.sign([user_sk, user2_sk]) + + tx.validate(b) + assert len(tx.inputs) == 1 + assert len(tx.outputs) == 1 + + def test_get_owned_ids_single_tx_single_output(self, b, user_sk, user_pk, alice): + from bigchaindb.common import crypto + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + + tx = Transaction.create([alice.public_key], [([user_pk], 1)]) + tx = tx.sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + owned_inputs_user2 = b.fastquery.get_outputs_by_public_key(user2_pk) + assert owned_inputs_user1 == [TransactionLink(tx.id, 0)] + assert owned_inputs_user2 == [] + + tx_transfer = Transaction.transfer(tx.to_inputs(), [([user2_pk], 1)], + asset_id=tx.id) + tx_transfer = tx_transfer.sign([user_sk]) + b.store_bulk_transactions([tx_transfer]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + owned_inputs_user2 = b.fastquery.get_outputs_by_public_key(user2_pk) + + assert owned_inputs_user1 == [TransactionLink(tx.id, 0)] + assert owned_inputs_user2 == [TransactionLink(tx_transfer.id, 0)] + + def test_get_owned_ids_single_tx_multiple_outputs(self, b, user_sk, + user_pk, alice): + from bigchaindb.common import crypto + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + + # create divisible asset + tx_create = Transaction.create([alice.public_key], [([user_pk], 1), ([user_pk], 1)]) + tx_create_signed = tx_create.sign([alice.private_key]) + b.store_bulk_transactions([tx_create_signed]) + + # get input + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + owned_inputs_user2 = b.fastquery.get_outputs_by_public_key(user2_pk) + + expected_owned_inputs_user1 = [TransactionLink(tx_create.id, 0), + TransactionLink(tx_create.id, 1)] + assert owned_inputs_user1 == expected_owned_inputs_user1 + assert owned_inputs_user2 == [] + + # transfer divisible asset divided in two outputs + tx_transfer = Transaction.transfer(tx_create.to_inputs(), + [([user2_pk], 1), ([user2_pk], 1)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + b.store_bulk_transactions([tx_transfer_signed]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + owned_inputs_user2 = b.fastquery.get_outputs_by_public_key(user2_pk) + assert owned_inputs_user1 == expected_owned_inputs_user1 + assert owned_inputs_user2 == [TransactionLink(tx_transfer.id, 0), + TransactionLink(tx_transfer.id, 1)] + + def test_get_owned_ids_multiple_owners(self, b, user_sk, user_pk, alice): + from bigchaindb.common import crypto + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + user3_sk, user3_pk = crypto.generate_key_pair() + + tx = Transaction.create([alice.public_key], [([user_pk, user2_pk], 1)]) + tx = tx.sign([alice.private_key]) + + b.store_bulk_transactions([tx]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + owned_inputs_user2 = b.fastquery.get_outputs_by_public_key(user_pk) + expected_owned_inputs_user1 = [TransactionLink(tx.id, 0)] + + assert owned_inputs_user1 == owned_inputs_user2 + assert owned_inputs_user1 == expected_owned_inputs_user1 + + tx = Transaction.transfer(tx.to_inputs(), [([user3_pk], 1)], + asset_id=tx.id) + tx = tx.sign([user_sk, user2_sk]) + b.store_bulk_transactions([tx]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + owned_inputs_user2 = b.fastquery.get_outputs_by_public_key(user2_pk) + spent_user1 = b.get_spent(tx.id, 0) + + assert owned_inputs_user1 == owned_inputs_user2 + assert not spent_user1 + + def test_get_spent_single_tx_single_output(self, b, user_sk, user_pk, alice): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + + tx = Transaction.create([alice.public_key], [([user_pk], 1)]) + tx = tx.sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk).pop() + + # check spents + input_txid = owned_inputs_user1.txid + spent_inputs_user1 = b.get_spent(input_txid, 0) + assert spent_inputs_user1 is None + + # create a transaction and send it + tx = Transaction.transfer(tx.to_inputs(), [([user2_pk], 1)], + asset_id=tx.id) + tx = tx.sign([user_sk]) + b.store_bulk_transactions([tx]) + + spent_inputs_user1 = b.get_spent(input_txid, 0) + assert spent_inputs_user1 == tx + + def test_get_spent_single_tx_multiple_outputs(self, b, user_sk, user_pk, alice): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + # create a new users + user2_sk, user2_pk = crypto.generate_key_pair() + + # create a divisible asset with 3 outputs + tx_create = Transaction.create([alice.public_key], + [([user_pk], 1), + ([user_pk], 1), + ([user_pk], 1)]) + tx_create_signed = tx_create.sign([alice.private_key]) + b.store_bulk_transactions([tx_create_signed]) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + + # check spents + for input_tx in owned_inputs_user1: + assert b.get_spent(input_tx.txid, input_tx.output) is None + + # transfer the first 2 inputs + tx_transfer = Transaction.transfer(tx_create.to_inputs()[:2], + [([user2_pk], 1), ([user2_pk], 1)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([user_sk]) + b.store_bulk_transactions([tx_transfer_signed]) + + # check that used inputs are marked as spent + for ffill in tx_create.to_inputs()[:2]: + spent_tx = b.get_spent(ffill.fulfills.txid, ffill.fulfills.output) + assert spent_tx == tx_transfer_signed + + # check if remaining transaction that was unspent is also perceived + # spendable by Planetmint + assert b.get_spent(tx_create.to_inputs()[2].fulfills.txid, 2) is None + + def test_get_spent_multiple_owners(self, b, user_sk, user_pk, alice): + from bigchaindb.common import crypto + from bigchaindb.models import Transaction + + user2_sk, user2_pk = crypto.generate_key_pair() + user3_sk, user3_pk = crypto.generate_key_pair() + + transactions = [] + for i in range(3): + payload = {'somedata': i} + tx = Transaction.create([alice.public_key], [([user_pk, user2_pk], 1)], + payload) + tx = tx.sign([alice.private_key]) + transactions.append(tx) + + b.store_bulk_transactions(transactions) + + owned_inputs_user1 = b.fastquery.get_outputs_by_public_key(user_pk) + # check spents + for input_tx in owned_inputs_user1: + assert b.get_spent(input_tx.txid, input_tx.output) is None + + # create a transaction + tx = Transaction.transfer(transactions[0].to_inputs(), + [([user3_pk], 1)], + asset_id=transactions[0].id) + tx = tx.sign([user_sk, user2_sk]) + b.store_bulk_transactions([tx]) + + # check that used inputs are marked as spent + assert b.get_spent(transactions[0].id, 0) == tx + # check that the other remain marked as unspent + for unspent in transactions[1:]: + assert b.get_spent(unspent.id, 0) is None + + +def test_get_outputs_filtered_only_unspent(): + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.lib import BigchainDB + + go = 'bigchaindb.fastquery.FastQuery.get_outputs_by_public_key' + with patch(go) as get_outputs: + get_outputs.return_value = [TransactionLink('a', 1), + TransactionLink('b', 2)] + fs = 'bigchaindb.fastquery.FastQuery.filter_spent_outputs' + with patch(fs) as filter_spent: + filter_spent.return_value = [TransactionLink('b', 2)] + out = BigchainDB).get_outputs_filtered('abc', spent=False) + get_outputs.assert_called_once_with('abc') + assert out == [TransactionLink('b', 2)] + + +def test_get_outputs_filtered_only_spent(): + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.lib import BigchainDB + go = 'bigchaindb.fastquery.FastQuery.get_outputs_by_public_key' + with patch(go) as get_outputs: + get_outputs.return_value = [TransactionLink('a', 1), + TransactionLink('b', 2)] + fs = 'bigchaindb.fastquery.FastQuery.filter_unspent_outputs' + with patch(fs) as filter_spent: + filter_spent.return_value = [TransactionLink('b', 2)] + out = BigchainDB).get_outputs_filtered('abc', spent=True) + get_outputs.assert_called_once_with('abc') + assert out == [TransactionLink('b', 2)] + + +@patch('bigchaindb.fastquery.FastQuery.filter_unspent_outputs') +@patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') +def test_get_outputs_filtered(filter_spent, filter_unspent): + from bigchaindb.common.transaction import TransactionLink + from bigchaindb.lib import BigchainDB + + go = 'bigchaindb.fastquery.FastQuery.get_outputs_by_public_key' + with patch(go) as get_outputs: + get_outputs.return_value = [TransactionLink('a', 1), + TransactionLink('b', 2)] + out = BigchainDB).get_outputs_filtered('abc') + get_outputs.assert_called_once_with('abc') + filter_spent.assert_not_called() + filter_unspent.assert_not_called() + assert out == get_outputs.return_value + + +def test_cant_spend_same_input_twice_in_tx(b, alice): + """Recreate duplicated fulfillments bug + https://github.com/bigchaindb/bigchaindb/issues/1099 + """ + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import DoubleSpend + + # create a divisible asset + tx_create = Transaction.create([alice.public_key], [([alice.public_key], 100)]) + tx_create_signed = tx_create.sign([alice.private_key]) + assert b.validate_transaction(tx_create_signed) == tx_create_signed + + b.store_bulk_transactions([tx_create_signed]) + + # Create a transfer transaction with duplicated fulfillments + dup_inputs = tx_create.to_inputs() + tx_create.to_inputs() + tx_transfer = Transaction.transfer(dup_inputs, [([alice.public_key], 200)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([alice.private_key]) + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) + + +def test_transaction_unicode(b, alice): + import copy + from bigchaindb.common.utils import serialize + from bigchaindb.models import Transaction + + # http://www.fileformat.info/info/unicode/char/1f37a/index.htm + beer_python = {'beer': '\N{BEER MUG}'} + beer_json = '{"beer":"\N{BEER MUG}"}' + + tx = (Transaction.create([alice.public_key], [([alice.public_key], 100)], beer_python) + ).sign([alice.private_key]) + + tx_1 = copy.deepcopy(tx) + b.store_bulk_transactions([tx]) + + assert beer_json in serialize(tx_1.to_dict()) diff --git a/tests/elections/__init__.py b/tests/elections/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/elections/test_election.py b/tests/elections/test_election.py new file mode 100644 index 0000000..bb70ff1 --- /dev/null +++ b/tests/elections/test_election.py @@ -0,0 +1,264 @@ +import pytest + +from tests.utils import generate_election, generate_validators + +from bigchaindb.lib import Block +from bigchaindb.elections.election import Election +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection +from bigchaindb.upsert_validator.validator_election import ValidatorElection + + +@pytest.mark.bdb +def test_process_block_concludes_all_elections(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + txs = [election] + total_votes = votes + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election'], + voter_keys) + txs += [election] + total_votes += votes + + b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 1, txs) + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.process_block(b, 2, total_votes) + + validators = b.get_validators() + assert len(validators) == 5 + assert new_validator['storage'] in validators + + chain = b.get_latest_abci_chain() + assert chain + assert chain == { + 'height': 2, + 'is_synced': False, + 'chain_id': 'chain-X-migrated-at-height-1', + } + + for tx in txs: + assert b.get_election(tx.id)['is_concluded'] + + +@pytest.mark.bdb +def test_process_block_approves_only_one_validator_update(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election'], + voter_keys) + txs = [election] + total_votes = votes + + another_validator = generate_validators([1])[0] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + another_validator['election'], + voter_keys) + txs += [election] + total_votes += votes + + Election.process_block(b, 1, txs) + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.process_block(b, 2, total_votes) + + validators = b.get_validators() + assert len(validators) == 5 + assert new_validator['storage'] in validators + assert another_validator['storage'] not in validators + + assert b.get_election(txs[0].id)['is_concluded'] + assert not b.get_election(txs[1].id)['is_concluded'] + + +@pytest.mark.bdb +def test_process_block_approves_after_pending_validator_update(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election'], + voter_keys) + txs = [election] + total_votes = votes + + another_validator = generate_validators([1])[0] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + another_validator['election'], + voter_keys) + txs += [election] + total_votes += votes + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + txs += [election] + total_votes += votes + + b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 1, txs) + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.process_block(b, 2, total_votes) + + validators = b.get_validators() + assert len(validators) == 5 + assert new_validator['storage'] in validators + assert another_validator['storage'] not in validators + + assert b.get_election(txs[0].id)['is_concluded'] + assert not b.get_election(txs[1].id)['is_concluded'] + assert b.get_election(txs[2].id)['is_concluded'] + + assert b.get_latest_abci_chain() == {'height': 2, + 'chain_id': 'chain-X-migrated-at-height-1', + 'is_synced': False} + + +@pytest.mark.bdb +def test_process_block_does_not_approve_after_validator_update(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election'], + voter_keys) + txs = [election] + total_votes = votes + + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + Election.process_block(b, 1, txs) + b.store_bulk_transactions(txs) + + second_election, second_votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + Election.process_block(b, 2, total_votes + [second_election]) + + b.store_block(Block(height=2, + transactions=[v.id for v in total_votes + [second_election]], + app_hash='')._asdict()) + + b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 3, second_votes) + + assert not b.get_election(second_election.id)['is_concluded'] + assert b.get_latest_abci_chain() == {'height': 1, + 'chain_id': 'chain-X', + 'is_synced': True} + + +@pytest.mark.bdb +def test_process_block_applies_only_one_migration(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + txs = [election] + total_votes = votes + + election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + txs += [election] + total_votes += votes + + b.store_abci_chain(1, 'chain-X') + Election.process_block(b, 1, txs) + b.store_block(Block(height=1, + transactions=[tx.id for tx in txs], + app_hash='')._asdict()) + b.store_bulk_transactions(txs) + + Election.process_block(b, 1, total_votes) + chain = b.get_latest_abci_chain() + assert chain + assert chain == { + 'height': 2, + 'is_synced': False, + 'chain_id': 'chain-X-migrated-at-height-1', + } + + assert b.get_election(txs[0].id)['is_concluded'] + assert not b.get_election(txs[1].id)['is_concluded'] + + +def test_process_block_gracefully_handles_empty_block(b): + Election.process_block(b, 1, []) diff --git a/tests/migrations/test_migration_election.py b/tests/migrations/test_migration_election.py new file mode 100644 index 0000000..b811903 --- /dev/null +++ b/tests/migrations/test_migration_election.py @@ -0,0 +1,9 @@ +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection + + +def test_valid_migration_election(b_mock, node_key): + voters = ChainMigrationElection.recipients(b_mock) + election = ChainMigrationElection.generate([node_key.public_key], + voters, + {}, None).sign([node_key.private_key]) + assert election.validate(b_mock) diff --git a/tests/tendermint/__init__.py b/tests/tendermint/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tendermint/conftest.py b/tests/tendermint/conftest.py new file mode 100644 index 0000000..aab19ec --- /dev/null +++ b/tests/tendermint/conftest.py @@ -0,0 +1,23 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest +import codecs + +from abci import types_v0_31_5 as types + + +@pytest.fixture +def validator_pub_key(): + return 'B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014' + + +@pytest.fixture +def init_chain_request(): + pk = codecs.decode(b'VAgFZtYw8bNR5TMZHFOBDWk9cAmEu3/c6JgRBmddbbI=', + 'base64') + val_a = types.ValidatorUpdate(power=10, + pub_key=types.PubKey(type='ed25519', data=pk)) + return types.RequestInitChain(validators=[val_a]) diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py new file mode 100644 index 0000000..6cfcdcd --- /dev/null +++ b/tests/tendermint/test_core.py @@ -0,0 +1,544 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import json +import pytest +import random + +from abci import types_v0_31_5 as types + +from bigchaindb import App +from bigchaindb.backend.localmongodb import query +from bigchaindb.common.crypto import generate_key_pair +from bigchaindb.core import (CodeTypeOk, + CodeTypeError, + rollback) +from bigchaindb.elections.election import Election +from bigchaindb.lib import Block +from bigchaindb.migrations.chain_migration_election import ChainMigrationElection +from bigchaindb.upsert_validator.validator_election import ValidatorElection +from bigchaindb.upsert_validator.validator_utils import new_validator_set +from bigchaindb.tendermint_utils import public_key_to_base64 +from bigchaindb.version import __tm_supported_versions__ + +from tests.utils import generate_election, generate_validators + + +pytestmark = pytest.mark.bdb + + +def encode_tx_to_bytes(transaction): + return json.dumps(transaction.to_dict()).encode('utf8') + + +def generate_address(): + return ''.join(random.choices('1,2,3,4,5,6,7,8,9,A,B,C,D,E,F'.split(','), + k=40)).encode() + + +def generate_validator(): + pk, _ = generate_key_pair() + pub_key = types.PubKey(type='ed25519', data=pk.encode()) + val = types.ValidatorUpdate(power=10, pub_key=pub_key) + return val + + +def generate_init_chain_request(chain_id, vals=None): + vals = vals if vals is not None else [generate_validator()] + return types.RequestInitChain(validators=vals, chain_id=chain_id) + + +def test_init_chain_successfully_registers_chain(a, b): + request = generate_init_chain_request('chain-XYZ') + res = App(a, b).init_chain(request) + assert res == types.ResponseInitChain() + chain = query.get_latest_abci_chain(b.connection) + assert chain == {'height': 0, 'chain_id': 'chain-XYZ', 'is_synced': True} + assert query.get_latest_block(b.connection) == { + 'height': 0, + 'app_hash': '', + 'transactions': [], + } + + +def test_init_chain_ignores_invalid_init_chain_requests(a, b): + validators = [generate_validator()] + request = generate_init_chain_request('chain-XYZ', validators) + res = App(a, b).init_chain(request) + assert res == types.ResponseInitChain() + + validator_set = query.get_validator_set(b.connection) + + invalid_requests = [ + request, # the same request again + # different validator set + generate_init_chain_request('chain-XYZ'), + # different chain ID + generate_init_chain_request('chain-ABC', validators), + ] + for r in invalid_requests: + with pytest.raises(SystemExit): + App(a, b).init_chain(r) + # assert nothing changed - neither validator set, nor chain ID + new_validator_set = query.get_validator_set(b.connection) + assert new_validator_set == validator_set + new_chain_id = query.get_latest_abci_chain(b.connection)['chain_id'] + assert new_chain_id == 'chain-XYZ' + assert query.get_latest_block(b.connection) == { + 'height': 0, + 'app_hash': '', + 'transactions': [], + } + + +def test_init_chain_recognizes_new_chain_after_migration(a, b): + validators = [generate_validator()] + request = generate_init_chain_request('chain-XYZ', validators) + res = App(a, b).init_chain(request) + assert res == types.ResponseInitChain() + + validator_set = query.get_validator_set(b.connection)['validators'] + + # simulate a migration + query.store_block(b.connection, Block(app_hash='', height=1, + transactions=[])._asdict()) + b.migrate_abci_chain() + + # the same or other mismatching requests are ignored + invalid_requests = [ + request, + generate_init_chain_request('unknown', validators), + generate_init_chain_request('chain-XYZ'), + generate_init_chain_request('chain-XYZ-migrated-at-height-1'), + ] + for r in invalid_requests: + with pytest.raises(SystemExit): + App(a, b).init_chain(r) + assert query.get_latest_abci_chain(b.connection) == { + 'chain_id': 'chain-XYZ-migrated-at-height-1', + 'is_synced': False, + 'height': 2, + } + new_validator_set = query.get_validator_set(b.connection)['validators'] + assert new_validator_set == validator_set + + # a request with the matching chain ID and matching validator set + # completes the migration + request = generate_init_chain_request('chain-XYZ-migrated-at-height-1', + validators) + res = App(a, b).init_chain(request) + assert res == types.ResponseInitChain() + assert query.get_latest_abci_chain(b.connection) == { + 'chain_id': 'chain-XYZ-migrated-at-height-1', + 'is_synced': True, + 'height': 2, + } + assert query.get_latest_block(b.connection) == { + 'height': 2, + 'app_hash': '', + 'transactions': [], + } + + # requests with old chain ID and other requests are ignored + invalid_requests = [ + request, + generate_init_chain_request('chain-XYZ', validators), + generate_init_chain_request('chain-XYZ-migrated-at-height-1'), + ] + for r in invalid_requests: + with pytest.raises(SystemExit): + App(a, b).init_chain(r) + assert query.get_latest_abci_chain(b.connection) == { + 'chain_id': 'chain-XYZ-migrated-at-height-1', + 'is_synced': True, + 'height': 2, + } + new_validator_set = query.get_validator_set(b.connection)['validators'] + assert new_validator_set == validator_set + assert query.get_latest_block(b.connection) == { + 'height': 2, + 'app_hash': '', + 'transactions': [], + } + + +def test_info(a, b): + r = types.RequestInfo(version=__tm_supported_versions__[0]) + app = App(a, b) + + res = app.info(r) + assert res.last_block_height == 0 + assert res.last_block_app_hash == b'' + + b.store_block(Block(app_hash='1', height=1, transactions=[])._asdict()) + res = app.info(r) + assert res.last_block_height == 1 + assert res.last_block_app_hash == b'1' + + # simulate a migration and assert the height is shifted + b.store_abci_chain(2, 'chain-XYZ') + app = App(a, b) + b.store_block(Block(app_hash='2', height=2, transactions=[])._asdict()) + res = app.info(r) + assert res.last_block_height == 0 + assert res.last_block_app_hash == b'2' + + b.store_block(Block(app_hash='3', height=3, transactions=[])._asdict()) + res = app.info(r) + assert res.last_block_height == 1 + assert res.last_block_app_hash == b'3' + + # it's always the latest migration that is taken into account + b.store_abci_chain(4, 'chain-XYZ-new') + app = App(a, b) + b.store_block(Block(app_hash='4', height=4, transactions=[])._asdict()) + res = app.info(r) + assert res.last_block_height == 0 + assert res.last_block_app_hash == b'4' + + +def test_check_tx__signed_create_is_ok(a, b): + from bigchaindb import App + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + bob = generate_key_pair() + + tx = Transaction.create([alice.public_key], + [([bob.public_key], 1)])\ + .sign([alice.private_key]) + + app = App(a, b) + result = app.check_tx(encode_tx_to_bytes(tx)) + assert result.code == CodeTypeOk + + +def test_check_tx__unsigned_create_is_error(a, b): + from bigchaindb import App + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + bob = generate_key_pair() + + tx = Transaction.create([alice.public_key], + [([bob.public_key], 1)]) + + app = App(a, b) + result = app.check_tx(encode_tx_to_bytes(tx)) + assert result.code == CodeTypeError + + +def test_deliver_tx__valid_create_updates_db_and_emits_event(a, b, init_chain_request): + import multiprocessing as mp + from bigchaindb import App + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + bob = generate_key_pair() + events = mp.Queue() + + tx = Transaction.create([alice.public_key], + [([bob.public_key], 1)])\ + .sign([alice.private_key]) + + app = App(a, b, events) + + app.init_chain(init_chain_request) + + begin_block = types.RequestBeginBlock() + app.begin_block(begin_block) + + result = app.deliver_tx(encode_tx_to_bytes(tx)) + assert result.code == CodeTypeOk + + app.end_block(types.RequestEndBlock(height=99)) + app.commit() + assert b.get_transaction(tx.id).id == tx.id + block_event = events.get() + assert block_event.data['transactions'] == [tx] + + # unspent_outputs = b.get_unspent_outputs() + # unspent_output = next(unspent_outputs) + # expected_unspent_output = next(tx.unspent_outputs)._asdict() + # assert unspent_output == expected_unspent_output + # with pytest.raises(StopIteration): + # next(unspent_outputs) + + +def test_deliver_tx__double_spend_fails(a, b, init_chain_request): + from bigchaindb import App + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + bob = generate_key_pair() + + tx = Transaction.create([alice.public_key], + [([bob.public_key], 1)])\ + .sign([alice.private_key]) + + app = App(a, b) + app.init_chain(init_chain_request) + + begin_block = types.RequestBeginBlock() + app.begin_block(begin_block) + + result = app.deliver_tx(encode_tx_to_bytes(tx)) + assert result.code == CodeTypeOk + + app.end_block(types.RequestEndBlock(height=99)) + app.commit() + + assert b.get_transaction(tx.id).id == tx.id + result = app.deliver_tx(encode_tx_to_bytes(tx)) + assert result.code == CodeTypeError + + +def test_deliver_transfer_tx__double_spend_fails(a, b, init_chain_request): + from bigchaindb import App + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + app = App(a, b) + app.init_chain(init_chain_request) + + begin_block = types.RequestBeginBlock() + app.begin_block(begin_block) + + alice = generate_key_pair() + bob = generate_key_pair() + carly = generate_key_pair() + + asset = { + 'msg': 'live long and prosper' + } + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=asset)\ + .sign([alice.private_key]) + + result = app.deliver_tx(encode_tx_to_bytes(tx)) + assert result.code == CodeTypeOk + + tx_transfer = Transaction.transfer(tx.to_inputs(), + [([bob.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + result = app.deliver_tx(encode_tx_to_bytes(tx_transfer)) + assert result.code == CodeTypeOk + + double_spend = Transaction.transfer(tx.to_inputs(), + [([carly.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + result = app.deliver_tx(encode_tx_to_bytes(double_spend)) + assert result.code == CodeTypeError + + +def test_end_block_return_validator_updates(a, b, init_chain_request): + app = App(a, b) + app.init_chain(init_chain_request) + + begin_block = types.RequestBeginBlock() + app.begin_block(begin_block) + + # generate a block containing a concluded validator election + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + + new_validator = generate_validators([1])[0] + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election'], + voter_keys) + b.store_block(Block(height=1, transactions=[election.id], + app_hash='')._asdict()) + b.store_bulk_transactions([election]) + Election.process_block(b, 1, [election]) + + app.block_transactions = votes + + resp = app.end_block(types.RequestEndBlock(height=2)) + assert resp.validator_updates[0].power == new_validator['election']['power'] + expected = bytes.fromhex(new_validator['election']['public_key']['value']) + assert expected == resp.validator_updates[0].pub_key.data + + +def test_store_pre_commit_state_in_end_block(a, b, alice, init_chain_request): + from bigchaindb import App + from bigchaindb.backend import query + from bigchaindb.models import Transaction + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset={'msg': 'live long and prosper'})\ + .sign([alice.private_key]) + + app = App(a, b) + app.init_chain(init_chain_request) + + begin_block = types.RequestBeginBlock() + app.begin_block(begin_block) + app.deliver_tx(encode_tx_to_bytes(tx)) + app.end_block(types.RequestEndBlock(height=99)) + + resp = query.get_pre_commit_state(b.connection) + assert resp['height'] == 99 + assert resp['transactions'] == [tx.id] + + app.begin_block(begin_block) + app.deliver_tx(encode_tx_to_bytes(tx)) + app.end_block(types.RequestEndBlock(height=100)) + resp = query.get_pre_commit_state(b.connection) + assert resp['height'] == 100 + assert resp['transactions'] == [tx.id] + + # simulate a chain migration and assert the height is shifted + b.store_abci_chain(100, 'new-chain') + app = App(a, b) + app.begin_block(begin_block) + app.deliver_tx(encode_tx_to_bytes(tx)) + app.end_block(types.RequestEndBlock(height=1)) + resp = query.get_pre_commit_state(b.connection) + assert resp['height'] == 101 + assert resp['transactions'] == [tx.id] + + +def test_rollback_pre_commit_state_after_crash(b): + validators = generate_validators([1] * 4) + b.store_validator_set(1, [v['storage'] for v in validators]) + b.store_block(Block(height=1, transactions=[], app_hash='')._asdict()) + + public_key = validators[0]['public_key'] + private_key = validators[0]['private_key'] + voter_keys = [v['private_key'] for v in validators] + + migration_election, votes = generate_election(b, + ChainMigrationElection, + public_key, private_key, + {}, + voter_keys) + + total_votes = votes + txs = [migration_election, *votes] + + new_validator = generate_validators([1])[0] + validator_election, votes = generate_election(b, + ValidatorElection, + public_key, private_key, + new_validator['election'], + voter_keys) + + total_votes += votes + txs += [validator_election, *votes] + + b.store_bulk_transactions(txs) + b.store_abci_chain(2, 'new_chain') + b.store_validator_set(2, [v['storage'] for v in validators]) + # TODO change to `4` when upgrading to Tendermint 0.22.4. + b.store_validator_set(3, [new_validator['storage']]) + b.store_election(migration_election.id, 2, is_concluded=False) + b.store_election(validator_election.id, 2, is_concluded=True) + + # no pre-commit state + rollback(b) + + for tx in txs: + assert b.get_transaction(tx.id) + assert b.get_latest_abci_chain() + assert len(b.get_validator_change()['validators']) == 1 + assert b.get_election(migration_election.id) + assert b.get_election(validator_election.id) + + b.store_pre_commit_state({'height': 2, 'transactions': [tx.id for tx in txs]}) + + rollback(b) + + for tx in txs: + assert not b.get_transaction(tx.id) + assert not b.get_latest_abci_chain() + assert len(b.get_validator_change()['validators']) == 4 + assert len(b.get_validator_change(2)['validators']) == 4 + assert not b.get_election(migration_election.id) + assert not b.get_election(validator_election.id) + + +def test_new_validator_set(b): + node1 = {'public_key': {'type': 'ed25519-base64', + 'value': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='}, + 'voting_power': 10} + node1_new_power = {'public_key': {'value': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034', + 'type': 'ed25519-base16'}, + 'power': 20} + node2 = {'public_key': {'value': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3', + 'type': 'ed25519-base16'}, + 'power': 10} + + validators = [node1] + updates = [node1_new_power, node2] + b.store_validator_set(1, validators) + updated_validator_set = new_validator_set(b.get_validators(1), updates) + + updated_validators = [] + for u in updates: + updated_validators.append({'public_key': {'type': 'ed25519-base64', + 'value': public_key_to_base64(u['public_key']['value'])}, + 'voting_power': u['power']}) + + assert updated_validator_set == updated_validators + + +def test_info_aborts_if_chain_is_not_synced(a, b): + b.store_abci_chain(0, 'chain-XYZ', False) + + with pytest.raises(SystemExit): + App(a, b).info(types.RequestInfo()) + + +def test_check_tx_aborts_if_chain_is_not_synced(a, b): + b.store_abci_chain(0, 'chain-XYZ', False) + + with pytest.raises(SystemExit): + App(a, b).check_tx('some bytes') + + +def test_begin_aborts_if_chain_is_not_synced(a, b): + b.store_abci_chain(0, 'chain-XYZ', False) + + with pytest.raises(SystemExit): + App(a, b).info(types.RequestBeginBlock()) + + +def test_deliver_tx_aborts_if_chain_is_not_synced(a, b): + b.store_abci_chain(0, 'chain-XYZ', False) + + with pytest.raises(SystemExit): + App(a, b).deliver_tx('some bytes') + + +def test_end_block_aborts_if_chain_is_not_synced(a, b): + b.store_abci_chain(0, 'chain-XYZ', False) + + with pytest.raises(SystemExit): + App(a, b).info(types.RequestEndBlock()) + + +def test_commit_aborts_if_chain_is_not_synced(a, b): + b.store_abci_chain(0, 'chain-XYZ', False) + + with pytest.raises(SystemExit): + App(a, b).commit() diff --git a/tests/tendermint/test_fastquery.py b/tests/tendermint/test_fastquery.py new file mode 100644 index 0000000..4d92ee6 --- /dev/null +++ b/tests/tendermint/test_fastquery.py @@ -0,0 +1,120 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + +from bigchaindb.common.transaction import TransactionLink +from bigchaindb.models import Transaction + + +pytestmark = pytest.mark.bdb + + +@pytest.fixture +def txns(b, user_pk, user_sk, user2_pk, user2_sk): + txs = [Transaction.create([user_pk], [([user2_pk], 1)]).sign([user_sk]), + Transaction.create([user2_pk], [([user_pk], 1)]).sign([user2_sk]), + Transaction.create([user_pk], [([user_pk], 1), ([user2_pk], 1)]) + .sign([user_sk])] + b.store_bulk_transactions(txs) + return txs + + +def test_get_outputs_by_public_key(b, user_pk, user2_pk, txns): + assert b.fastquery.get_outputs_by_public_key(user_pk) == [ + TransactionLink(txns[1].id, 0), + TransactionLink(txns[2].id, 0) + ] + assert b.fastquery.get_outputs_by_public_key(user2_pk) == [ + TransactionLink(txns[0].id, 0), + TransactionLink(txns[2].id, 1), + ] + + +def test_filter_spent_outputs(b, user_pk, user_sk): + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 2) + tx1.sign([user_sk]) + + inputs = tx1.to_inputs() + + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx2.sign([user_sk]) + + # tx2 produces a new unspent. inputs[1] remains unspent. + b.store_bulk_transactions([tx1, tx2]) + + outputs = b.fastquery.get_outputs_by_public_key(user_pk) + unspents = b.fastquery.filter_spent_outputs(outputs) + + assert set(unsp for unsp in unspents) == { + inputs[1].fulfills, + tx2.to_inputs()[0].fulfills, + } + + +def test_filter_unspent_outputs(b, user_pk, user_sk): + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 2) + tx1.sign([user_sk]) + + inputs = tx1.to_inputs() + + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx2.sign([user_sk]) + + # tx2 produces a new unspent. input[1] remains unspent. + b.store_bulk_transactions([tx1, tx2]) + + outputs = b.fastquery.get_outputs_by_public_key(user_pk) + spents = b.fastquery.filter_unspent_outputs(outputs) + + assert set(sp for sp in spents) == { + inputs[0].fulfills, + } + + +def test_outputs_query_key_order(b, user_pk, user_sk, user2_pk, user2_sk): + from bigchaindb import backend + from bigchaindb.backend import connect + + tx1 = Transaction.create([user_pk], + [([user_pk], 3), ([user_pk], 2), ([user_pk], 1)])\ + .sign([user_sk]) + b.store_bulk_transactions([tx1]) + + inputs = tx1.to_inputs() + tx2 = Transaction.transfer([inputs[1]], [([user2_pk], 2)], tx1.id).sign([user_sk]) + assert tx2.validate(b) + + tx2_dict = tx2.to_dict() + fulfills = tx2_dict['inputs'][0]['fulfills'] + tx2_dict['inputs'][0]['fulfills'] = {'transaction_id': fulfills['transaction_id'], + 'output_index': fulfills['output_index']} + backend.query.store_transactions(b.connection, [tx2_dict]) + + outputs = b.get_outputs_filtered(user_pk, spent=False) + assert len(outputs) == 2 + + outputs = b.get_outputs_filtered(user2_pk, spent=False) + assert len(outputs) == 1 + + # clean the transaction, metdata and asset collection + conn = connect() + conn.run(conn.collection('transactions').delete_many({})) + conn.run(conn.collection('metadata').delete_many({})) + conn.run(conn.collection('assets').delete_many({})) + + b.store_bulk_transactions([tx1]) + tx2_dict = tx2.to_dict() + tx2_dict['inputs'][0]['fulfills'] = {'output_index': fulfills['output_index'], + 'transaction_id': fulfills['transaction_id']} + + backend.query.store_transactions(b.connection, [tx2_dict]) + outputs = b.get_outputs_filtered(user_pk, spent=False) + assert len(outputs) == 2 + + outputs = b.get_outputs_filtered(user2_pk, spent=False) + assert len(outputs) == 1 diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py new file mode 100644 index 0000000..e903142 --- /dev/null +++ b/tests/tendermint/test_integration.py @@ -0,0 +1,157 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import codecs + +from abci import types_v0_31_5 as types +import json +import pytest + + +from abci.server import ProtocolHandler +from abci.encoding import read_messages + +from bigchaindb.common.transaction_mode_types import BROADCAST_TX_COMMIT, BROADCAST_TX_SYNC +from bigchaindb.version import __tm_supported_versions__ +from io import BytesIO + + +@pytest.mark.bdb +def test_app(a, b, init_chain_request): + from bigchaindb import App + from bigchaindb.tendermint_utils import calculate_hash + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.models import Transaction + + app = App(a, b) + p = ProtocolHandler(app) + + data = p.process('info', + types.Request(info=types.RequestInfo(version=__tm_supported_versions__[0]))) + res = next(read_messages(BytesIO(data), types.Response)) + assert res + assert res.info.last_block_app_hash == b'' + assert res.info.last_block_height == 0 + assert not b.get_latest_block() + + p.process('init_chain', types.Request(init_chain=init_chain_request)) + block0 = b.get_latest_block() + assert block0 + assert block0['height'] == 0 + assert block0['app_hash'] == '' + + pk = codecs.encode(init_chain_request.validators[0].pub_key.data, 'base64').decode().strip('\n') + [validator] = b.get_validators(height=1) + assert validator['public_key']['value'] == pk + assert validator['voting_power'] == 10 + + alice = generate_key_pair() + bob = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([bob.public_key], 1)])\ + .sign([alice.private_key]) + etxn = json.dumps(tx.to_dict()).encode('utf8') + + r = types.Request(check_tx=types.RequestCheckTx(tx=etxn)) + data = p.process('check_tx', r) + res = next(read_messages(BytesIO(data), types.Response)) + assert res + assert res.check_tx.code == 0 + + r = types.Request() + r.begin_block.hash = b'' + p.process('begin_block', r) + + r = types.Request(deliver_tx=types.RequestDeliverTx(tx=etxn)) + data = p.process('deliver_tx', r) + res = next(read_messages(BytesIO(data), types.Response)) + assert res + assert res.deliver_tx.code == 0 + + new_block_txn_hash = calculate_hash([tx.id]) + + r = types.Request(end_block=types.RequestEndBlock(height=1)) + data = p.process('end_block', r) + res = next(read_messages(BytesIO(data), types.Response)) + assert res + assert 'end_block' == res.WhichOneof('value') + + new_block_hash = calculate_hash([block0['app_hash'], new_block_txn_hash]) + + data = p.process('commit', None) + res = next(read_messages(BytesIO(data), types.Response)) + assert res.commit.data == new_block_hash.encode('utf-8') + assert b.get_transaction(tx.id).id == tx.id + + block0 = b.get_latest_block() + assert block0 + assert block0['height'] == 1 + assert block0['app_hash'] == new_block_hash + + # empty block should not update height + r = types.Request() + r.begin_block.hash = new_block_hash.encode('utf-8') + p.process('begin_block', r) + + r = types.Request() + r.end_block.height = 2 + p.process('end_block', r) + + data = p.process('commit', None) + res = next(read_messages(BytesIO(data), types.Response)) + assert res.commit.data == new_block_hash.encode('utf-8') + + block0 = b.get_latest_block() + assert block0 + assert block0['height'] == 2 + + # when empty block is generated hash of previous block should be returned + assert block0['app_hash'] == new_block_hash + + +@pytest.mark.abci +def test_post_transaction_responses(tendermint_ws_url, b): + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.models import Transaction + + alice = generate_key_pair() + bob = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None)\ + .sign([alice.private_key]) + + code, message = b.write_transaction(tx, BROADCAST_TX_COMMIT) + assert code == 202 + + tx_transfer = Transaction.transfer(tx.to_inputs(), + [([bob.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + code, message = b.write_transaction(tx_transfer, BROADCAST_TX_COMMIT) + assert code == 202 + + carly = generate_key_pair() + double_spend = Transaction.transfer( + tx.to_inputs(), + [([carly.public_key], 1)], + asset_id=tx.id, + ).sign([alice.private_key]) + for mode in (BROADCAST_TX_SYNC, BROADCAST_TX_COMMIT): + code, message = b.write_transaction(double_spend, mode) + assert code == 500 + assert message == 'Transaction validation failed' + + +@pytest.mark.bdb +def test_exit_when_tm_ver_not_supported(a, b): + from bigchaindb import App + + app = App(a, b) + p = ProtocolHandler(app) + + with pytest.raises(SystemExit): + p.process('info', types.Request(info=types.RequestInfo(version='2'))) diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py new file mode 100644 index 0000000..7c79711 --- /dev/null +++ b/tests/tendermint/test_lib.py @@ -0,0 +1,490 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import os +from unittest.mock import patch + + +try: + from hashlib import sha3_256 +except ImportError: + # NOTE: needed for Python < 3.6 + from sha3 import sha3_256 + +import pytest +from pymongo import MongoClient + +from bigchaindb import backend +from bigchaindb.common.transaction_mode_types import (BROADCAST_TX_COMMIT, + BROADCAST_TX_ASYNC, + BROADCAST_TX_SYNC) +from bigchaindb.lib import Block + + +@pytest.mark.bdb +def test_asset_is_separated_from_transaciton(b): + import copy + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + bob = generate_key_pair() + + asset = {'Never gonna': ['give you up', + 'let you down', + 'run around' + 'desert you', + 'make you cry', + 'say goodbye', + 'tell a lie', + 'hurt you']} + + tx = Transaction.create([alice.public_key], + [([bob.public_key], 1)], + metadata=None, + asset=asset)\ + .sign([alice.private_key]) + + # with store_bulk_transactions we use `insert_many` where PyMongo + # automatically adds an `_id` field to the tx, therefore we need the + # deepcopy, for more info see: + # https://api.mongodb.com/python/current/faq.html#writes-and-ids + tx_dict = copy.deepcopy(tx.to_dict()) + + b.store_bulk_transactions([tx]) + assert 'asset' not in backend.query.get_transaction(b.connection, tx.id) + assert backend.query.get_asset(b.connection, tx.id)['data'] == asset + assert b.get_transaction(tx.id).to_dict() == tx_dict + + +@pytest.mark.bdb +def test_get_latest_block(b): + from bigchaindb.lib import Block + + for i in range(10): + app_hash = os.urandom(16).hex() + txn_id = os.urandom(16).hex() + block = Block(app_hash=app_hash, height=i, + transactions=[txn_id])._asdict() + b.store_block(block) + + block = b.get_latest_block() + assert block['height'] == 9 + + +@pytest.mark.bdb +@patch('bigchaindb.backend.query.get_block', return_value=None) +@patch('bigchaindb.Planetmint.get_latest_block', return_value={'height': 10}) +def test_get_empty_block(_0, _1, b): + assert b.get_block(5) == {'height': 5, 'transactions': []} + + +def test_validation_error(b): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None)\ + .sign([alice.private_key]).to_dict() + + tx['metadata'] = '' + assert not b.validate_transaction(tx) + + +@patch('requests.post') +def test_write_and_post_transaction(mock_post, b): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.tendermint_utils import encode_transaction + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None)\ + .sign([alice.private_key]).to_dict() + + tx = b.validate_transaction(tx) + b.write_transaction(tx, BROADCAST_TX_ASYNC) + + assert mock_post.called + args, kwargs = mock_post.call_args + assert BROADCAST_TX_ASYNC == kwargs['json']['method'] + encoded_tx = [encode_transaction(tx.to_dict())] + assert encoded_tx == kwargs['json']['params'] + + +@patch('requests.post') +@pytest.mark.parametrize('mode', [ + BROADCAST_TX_SYNC, + BROADCAST_TX_ASYNC, + BROADCAST_TX_COMMIT +]) +def test_post_transaction_valid_modes(mock_post, b, mode): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]).to_dict() + tx = b.validate_transaction(tx) + b.write_transaction(tx, mode) + + args, kwargs = mock_post.call_args + assert mode == kwargs['json']['method'] + + +def test_post_transaction_invalid_mode(b): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.common.exceptions import ValidationError + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]).to_dict() + tx = b.validate_transaction(tx) + with pytest.raises(ValidationError): + b.write_transaction(tx, 'nope') + + +@pytest.mark.bdb +def test_update_utxoset(b, signed_create_tx, signed_transfer_tx, db_context): + mongo_client = MongoClient(host=db_context.host, port=db_context.port) + b.update_utxoset(signed_create_tx) + utxoset = mongo_client[db_context.name]['utxos'] + assert utxoset.count_documents({}) == 1 + utxo = utxoset.find_one() + assert utxo['transaction_id'] == signed_create_tx.id + assert utxo['output_index'] == 0 + b.update_utxoset(signed_transfer_tx) + assert utxoset.count_documents({}) == 1 + utxo = utxoset.find_one() + assert utxo['transaction_id'] == signed_transfer_tx.id + assert utxo['output_index'] == 0 + + +@pytest.mark.bdb +def test_store_transaction(mocker, b, signed_create_tx, + signed_transfer_tx, db_context): + mocked_store_asset = mocker.patch('bigchaindb.backend.query.store_assets') + mocked_store_metadata = mocker.patch( + 'bigchaindb.backend.query.store_metadatas') + mocked_store_transaction = mocker.patch( + 'bigchaindb.backend.query.store_transactions') + b.store_bulk_transactions([signed_create_tx]) + # mongo_client = MongoClient(host=db_context.host, port=db_context.port) + # utxoset = mongo_client[db_context.name]['utxos'] + # assert utxoset.count_documents({}) == 1 + # utxo = utxoset.find_one() + # assert utxo['transaction_id'] == signed_create_tx.id + # assert utxo['output_index'] == 0 + + mocked_store_asset.assert_called_once_with( + b.connection, + [{'id': signed_create_tx.id, 'data': signed_create_tx.asset['data']}], + ) + mocked_store_metadata.assert_called_once_with( + b.connection, + [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}], + ) + mocked_store_transaction.assert_called_once_with( + b.connection, + [{k: v for k, v in signed_create_tx.to_dict().items() + if k not in ('asset', 'metadata')}], + ) + mocked_store_asset.reset_mock() + mocked_store_metadata.reset_mock() + mocked_store_transaction.reset_mock() + b.store_bulk_transactions([signed_transfer_tx]) + # assert utxoset.count_documents({}) == 1 + # utxo = utxoset.find_one() + # assert utxo['transaction_id'] == signed_transfer_tx.id + # assert utxo['output_index'] == 0 + assert not mocked_store_asset.called + mocked_store_metadata.asser_called_once_with( + b.connection, + [{'id': signed_transfer_tx.id, 'metadata': signed_transfer_tx.metadata}], + ) + mocked_store_transaction.assert_called_once_with( + b.connection, + [{k: v for k, v in signed_transfer_tx.to_dict().items() + if k != 'metadata'}], + ) + + +@pytest.mark.bdb +def test_store_bulk_transaction(mocker, b, signed_create_tx, + signed_transfer_tx, db_context): + mocked_store_assets = mocker.patch( + 'bigchaindb.backend.query.store_assets') + mocked_store_metadata = mocker.patch( + 'bigchaindb.backend.query.store_metadatas') + mocked_store_transactions = mocker.patch( + 'bigchaindb.backend.query.store_transactions') + b.store_bulk_transactions((signed_create_tx,)) + # mongo_client = MongoClient(host=db_context.host, port=db_context.port) + # utxoset = mongo_client[db_context.name]['utxos'] + # assert utxoset.count_documents({}) == 1 + # utxo = utxoset.find_one() + # assert utxo['transaction_id'] == signed_create_tx.id + # assert utxo['output_index'] == 0 + mocked_store_assets.assert_called_once_with( + b.connection, + [{'id': signed_create_tx.id, 'data': signed_create_tx.asset['data']}], + ) + mocked_store_metadata.assert_called_once_with( + b.connection, + [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}], + ) + mocked_store_transactions.assert_called_once_with( + b.connection, + [{k: v for k, v in signed_create_tx.to_dict().items() + if k not in ('asset', 'metadata')}], + ) + mocked_store_assets.reset_mock() + mocked_store_metadata.reset_mock() + mocked_store_transactions.reset_mock() + b.store_bulk_transactions((signed_transfer_tx,)) + # assert utxoset.count_documents({}) == 1 + # utxo = utxoset.find_one() + # assert utxo['transaction_id'] == signed_transfer_tx.id + # assert utxo['output_index'] == 0 + assert not mocked_store_assets.called + mocked_store_metadata.asser_called_once_with( + b.connection, + [{'id': signed_transfer_tx.id, + 'metadata': signed_transfer_tx.metadata}], + ) + mocked_store_transactions.assert_called_once_with( + b.connection, + [{k: v for k, v in signed_transfer_tx.to_dict().items() + if k != 'metadata'}], + ) + + +@pytest.mark.bdb +def test_delete_zero_unspent_outputs(b, utxoset): + unspent_outputs, utxo_collection = utxoset + delete_res = b.delete_unspent_outputs() + assert delete_res is None + assert utxo_collection.count_documents({}) == 3 + assert utxo_collection.count_documents( + {'$or': [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'b', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 1}, + ]} + ) == 3 + + +@pytest.mark.bdb +def test_delete_one_unspent_outputs(b, utxoset): + unspent_outputs, utxo_collection = utxoset + delete_res = b.delete_unspent_outputs(unspent_outputs[0]) + assert delete_res.raw_result['n'] == 1 + assert utxo_collection.count_documents( + {'$or': [ + {'transaction_id': 'a', 'output_index': 1}, + {'transaction_id': 'b', 'output_index': 0}, + ]} + ) == 2 + assert utxo_collection.count_documents( + {'transaction_id': 'a', 'output_index': 0}) == 0 + + +@pytest.mark.bdb +def test_delete_many_unspent_outputs(b, utxoset): + unspent_outputs, utxo_collection = utxoset + delete_res = b.delete_unspent_outputs(*unspent_outputs[::2]) + assert delete_res.raw_result['n'] == 2 + assert utxo_collection.count_documents( + {'$or': [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'b', 'output_index': 0}, + ]} + ) == 0 + assert utxo_collection.count_documents( + {'transaction_id': 'a', 'output_index': 1}) == 1 + + +@pytest.mark.bdb +def test_store_zero_unspent_output(b, utxo_collection): + res = b.store_unspent_outputs() + assert res is None + assert utxo_collection.count_documents({}) == 0 + + +@pytest.mark.bdb +def test_store_one_unspent_output(b, unspent_output_1, utxo_collection): + res = b.store_unspent_outputs(unspent_output_1) + assert res.acknowledged + assert len(res.inserted_ids) == 1 + assert utxo_collection.count_documents( + {'transaction_id': unspent_output_1['transaction_id'], + 'output_index': unspent_output_1['output_index']} + ) == 1 + + +@pytest.mark.bdb +def test_store_many_unspent_outputs(b, unspent_outputs, utxo_collection): + res = b.store_unspent_outputs(*unspent_outputs) + assert res.acknowledged + assert len(res.inserted_ids) == 3 + assert utxo_collection.count_documents( + {'transaction_id': unspent_outputs[0]['transaction_id']} + ) == 3 + + +def test_get_utxoset_merkle_root_when_no_utxo(b): + assert b.get_utxoset_merkle_root() == sha3_256(b'').hexdigest() + + +@pytest.mark.bdb +@pytest.mark.usefixture('utxoset') +def test_get_utxoset_merkle_root(b, utxoset): + expected_merkle_root = ( + '86d311c03115bf4d287f8449ca5828505432d69b82762d47077b1c00fe426eac') + merkle_root = b.get_utxoset_merkle_root() + assert merkle_root == expected_merkle_root + + +@pytest.mark.bdb +def test_get_spent_transaction_critical_double_spend(b, alice, bob, carol): + from bigchaindb.models import Transaction + from bigchaindb.exceptions import CriticalDoubleSpend + from bigchaindb.common.exceptions import DoubleSpend + + asset = {'test': 'asset'} + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=asset)\ + .sign([alice.private_key]) + + tx_transfer = Transaction.transfer(tx.to_inputs(), + [([bob.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + double_spend = Transaction.transfer(tx.to_inputs(), + [([carol.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + same_input_double_spend = Transaction.transfer(tx.to_inputs() + tx.to_inputs(), + [([bob.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + b.store_bulk_transactions([tx]) + + with pytest.raises(DoubleSpend): + same_input_double_spend.validate(b) + + assert b.get_spent(tx.id, tx_transfer.inputs[0].fulfills.output, [tx_transfer]) + + with pytest.raises(DoubleSpend): + b.get_spent(tx.id, tx_transfer.inputs[0].fulfills.output, + [tx_transfer, double_spend]) + + b.store_bulk_transactions([tx_transfer]) + + with pytest.raises(DoubleSpend): + b.get_spent(tx.id, tx_transfer.inputs[0].fulfills.output, [double_spend]) + + b.store_bulk_transactions([double_spend]) + + with pytest.raises(CriticalDoubleSpend): + b.get_spent(tx.id, tx_transfer.inputs[0].fulfills.output) + + +def test_validation_with_transaction_buffer(b): + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.models import Transaction + + priv_key, pub_key = generate_key_pair() + + create_tx = Transaction.create([pub_key], [([pub_key], 10)]).sign([priv_key]) + transfer_tx = Transaction.transfer(create_tx.to_inputs(), + [([pub_key], 10)], + asset_id=create_tx.id).sign([priv_key]) + double_spend = Transaction.transfer(create_tx.to_inputs(), + [([pub_key], 10)], + asset_id=create_tx.id).sign([priv_key]) + + assert b.is_valid_transaction(create_tx) + assert b.is_valid_transaction(transfer_tx, [create_tx]) + + assert not b.is_valid_transaction(create_tx, [create_tx]) + assert not b.is_valid_transaction(transfer_tx, [create_tx, transfer_tx]) + assert not b.is_valid_transaction(double_spend, [create_tx, transfer_tx]) + + +@pytest.mark.bdb +def test_migrate_abci_chain_yields_on_genesis(b): + b.migrate_abci_chain() + latest_chain = b.get_latest_abci_chain() + assert latest_chain is None + + +@pytest.mark.bdb +@pytest.mark.parametrize('chain,block_height,expected', [ + ( + (1, 'chain-XYZ', True), + 4, + {'height': 5, 'chain_id': 'chain-XYZ-migrated-at-height-4', + 'is_synced': False}, + ), + ( + (5, 'chain-XYZ-migrated-at-height-4', True), + 13, + {'height': 14, 'chain_id': 'chain-XYZ-migrated-at-height-13', + 'is_synced': False}, + ), +]) +def test_migrate_abci_chain_generates_new_chains(b, chain, block_height, + expected): + b.store_abci_chain(*chain) + b.store_block(Block(app_hash='', height=block_height, + transactions=[])._asdict()) + b.migrate_abci_chain() + latest_chain = b.get_latest_abci_chain() + assert latest_chain == expected + + +@pytest.mark.bdb +def test_get_spent_key_order(b, user_pk, user_sk, user2_pk, user2_sk): + from bigchaindb import backend + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.common.exceptions import DoubleSpend + + alice = generate_key_pair() + bob = generate_key_pair() + + tx1 = Transaction.create([user_pk], + [([alice.public_key], 3), ([user_pk], 2)], + asset=None)\ + .sign([user_sk]) + b.store_bulk_transactions([tx1]) + + inputs = tx1.to_inputs() + tx2 = Transaction.transfer([inputs[1]], [([user2_pk], 2)], tx1.id).sign([user_sk]) + assert tx2.validate(b) + + tx2_dict = tx2.to_dict() + fulfills = tx2_dict['inputs'][0]['fulfills'] + tx2_dict['inputs'][0]['fulfills'] = {'output_index': fulfills['output_index'], + 'transaction_id': fulfills['transaction_id']} + + backend.query.store_transactions(b.connection, [tx2_dict]) + + tx3 = Transaction.transfer([inputs[1]], [([bob.public_key], 2)], tx1.id).sign([user_sk]) + + with pytest.raises(DoubleSpend): + tx3.validate(b) diff --git a/tests/tendermint/test_utils.py b/tests/tendermint/test_utils.py new file mode 100644 index 0000000..c5a74fc --- /dev/null +++ b/tests/tendermint/test_utils.py @@ -0,0 +1,71 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import base64 +import json + +try: + from hashlib import sha3_256 +except ImportError: + from sha3 import sha3_256 + + +def test_encode_decode_transaction(b): + from bigchaindb.tendermint_utils import (encode_transaction, + decode_transaction) + + asset = { + 'value': 'key' + } + + encode_tx = encode_transaction(asset) + new_encode_tx = base64.b64encode(json.dumps(asset). + encode('utf8')).decode('utf8') + + assert encode_tx == new_encode_tx + + de64 = base64.b64decode(encode_tx) + assert asset == decode_transaction(de64) + + +def test_calculate_hash_no_key(b): + from bigchaindb.tendermint_utils import calculate_hash + + # pass an empty list + assert calculate_hash([]) == '' + + +# TODO test for the case of an empty list of hashes, and possibly other cases. +def test_merkleroot(): + from bigchaindb.tendermint_utils import merkleroot + hashes = [sha3_256(i.encode()).digest() for i in 'abc'] + assert merkleroot(hashes) == ( + '78c7c394d3158c218916b7ae0ebdea502e0f4e85c08e3b371e3dfd824d389fa3') + + +SAMPLE_PUBLIC_KEY = { + "address": "53DC09497A6ED73B342C78AB1E916076A03A8B95", + "pub_key": { + "type": "AC26791624DE60", + "value": "7S+T/do70jvneAq0M1so2X3M1iWTSuwtuSAr3nVpfEw=" + } +} + + +def test_convert_base64_public_key_to_address(): + from bigchaindb.tendermint_utils import public_key64_to_address + + address = public_key64_to_address(SAMPLE_PUBLIC_KEY['pub_key']['value']) + assert address == SAMPLE_PUBLIC_KEY['address'] + + +def test_public_key_encoding_decoding(): + from bigchaindb.tendermint_utils import (public_key_from_base64, + public_key_to_base64) + + public_key = public_key_from_base64(SAMPLE_PUBLIC_KEY['pub_key']['value']) + base64_public_key = public_key_to_base64(public_key) + + assert base64_public_key == SAMPLE_PUBLIC_KEY['pub_key']['value'] diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py new file mode 100644 index 0000000..1b092a6 --- /dev/null +++ b/tests/test_config_utils.py @@ -0,0 +1,332 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import copy +from unittest.mock import mock_open, patch + +import pytest + +import bigchaindb + + +ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config) + + +@pytest.fixture(scope='function', autouse=True) +def clean_config(monkeypatch, request): + original_config = copy.deepcopy(ORIGINAL_CONFIG) + backend = request.config.getoption('--database-backend') + original_config['database'] = bigchaindb._database_map[backend] + monkeypatch.setattr('bigchaindb.config', original_config) + + +def test_bigchain_instance_is_initialized_when_conf_provided(): + import bigchaindb + from bigchaindb import config_utils + assert 'CONFIGURED' not in bigchaindb.config + + config_utils.set_config({'database': {'backend': 'a'}}) + + assert bigchaindb.config['CONFIGURED'] is True + + +def test_load_validation_plugin_loads_default_rules_without_name(): + from bigchaindb import config_utils + from bigchaindb.validation import BaseValidationRules + + assert config_utils.load_validation_plugin() == BaseValidationRules + + +def test_load_validation_plugin_raises_with_unknown_name(): + from pkg_resources import ResolutionError + from bigchaindb import config_utils + + with pytest.raises(ResolutionError): + config_utils.load_validation_plugin('bogus') + + +def test_load_validation_plugin_raises_with_invalid_subclass(monkeypatch): + # Monkeypatch entry_point.load to return something other than a + # ValidationRules instance + from bigchaindb import config_utils + import time + monkeypatch.setattr(config_utils, + 'iter_entry_points', + lambda *args: [type('entry_point', (object, ), {'load': lambda: object})]) + + with pytest.raises(TypeError): + # Since the function is decorated with `lru_cache`, we need to + # "miss" the cache using a name that has not been used previously + config_utils.load_validation_plugin(str(time.time())) + + +def test_load_events_plugins(monkeypatch): + from bigchaindb import config_utils + monkeypatch.setattr(config_utils, + 'iter_entry_points', + lambda *args: [type('entry_point', (object, ), {'load': lambda: object})]) + + plugins = config_utils.load_events_plugins(['one', 'two']) + assert len(plugins) == 2 + + +def test_map_leafs_iterator(): + from bigchaindb import config_utils + + mapping = { + 'a': {'b': {'c': 1}, + 'd': {'z': 44}}, + 'b': {'d': 2}, + 'c': 3 + } + + result = config_utils.map_leafs(lambda x, path: x * 2, mapping) + assert result == { + 'a': {'b': {'c': 2}, + 'd': {'z': 88}}, + 'b': {'d': 4}, + 'c': 6 + } + + result = config_utils.map_leafs(lambda x, path: path, mapping) + assert result == { + 'a': {'b': {'c': ['a', 'b', 'c']}, + 'd': {'z': ['a', 'd', 'z']}}, + 'b': {'d': ['b', 'd']}, + 'c': ['c'] + } + + +def test_update_types(): + from bigchaindb import config_utils + + raw = { + 'a_string': 'test', + 'an_int': '42', + 'a_float': '3.14', + 'a_list': 'a:b:c', + } + + reference = { + 'a_string': 'test', + 'an_int': 42, + 'a_float': 3.14, + 'a_list': ['a', 'b', 'c'], + } + + result = config_utils.update_types(raw, reference) + assert result == reference + + +def test_env_config(monkeypatch): + monkeypatch.setattr('os.environ', {'PLANETMINT_DATABASE_HOST': 'test-host', + 'PLANETMINT_DATABASE_PORT': 'test-port'}) + + from bigchaindb import config_utils + + result = config_utils.env_config({'database': {'host': None, 'port': None}}) + expected = {'database': {'host': 'test-host', 'port': 'test-port'}} + + assert result == expected + + +def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): + # constants + DATABASE_HOST = 'test-host' + DATABASE_NAME = 'test-dbname' + DATABASE_PORT = 4242 + DATABASE_BACKEND = request.config.getoption('--database-backend') + SERVER_BIND = '1.2.3.4:56' + WSSERVER_SCHEME = 'ws' + WSSERVER_HOST = '1.2.3.4' + WSSERVER_PORT = 57 + WSSERVER_ADVERTISED_SCHEME = 'wss' + WSSERVER_ADVERTISED_HOST = 'a.b.c.d' + WSSERVER_ADVERTISED_PORT = 89 + LOG_FILE = '/somewhere/something.log' + + file_config = { + 'database': { + 'host': DATABASE_HOST + }, + 'log': { + 'level_console': 'debug', + }, + } + + monkeypatch.setattr('bigchaindb.config_utils.file_config', + lambda *args, **kwargs: file_config) + + monkeypatch.setattr('os.environ', { + 'PLANETMINT_DATABASE_NAME': DATABASE_NAME, + 'PLANETMINT_DATABASE_PORT': str(DATABASE_PORT), + 'PLANETMINT_DATABASE_BACKEND': DATABASE_BACKEND, + 'PLANETMINT_SERVER_BIND': SERVER_BIND, + 'PLANETMINT_WSSERVER_SCHEME': WSSERVER_SCHEME, + 'PLANETMINT_WSSERVER_HOST': WSSERVER_HOST, + 'PLANETMINT_WSSERVER_PORT': WSSERVER_PORT, + 'PLANETMINT_WSSERVER_ADVERTISED_SCHEME': WSSERVER_ADVERTISED_SCHEME, + 'PLANETMINT_WSSERVER_ADVERTISED_HOST': WSSERVER_ADVERTISED_HOST, + 'PLANETMINT_WSSERVER_ADVERTISED_PORT': WSSERVER_ADVERTISED_PORT, + 'PLANETMINT_LOG_FILE': LOG_FILE, + 'PLANETMINT_LOG_FILE': LOG_FILE, + 'PLANETMINT_DATABASE_CA_CERT': 'ca_cert', + 'PLANETMINT_DATABASE_CRLFILE': 'crlfile', + 'PLANETMINT_DATABASE_CERTFILE': 'certfile', + 'PLANETMINT_DATABASE_KEYFILE': 'keyfile', + 'PLANETMINT_DATABASE_KEYFILE_PASSPHRASE': 'passphrase', + }) + + import bigchaindb + from bigchaindb import config_utils + from bigchaindb.log import DEFAULT_LOGGING_CONFIG as log_config + config_utils.autoconfigure() + + database_mongodb = { + 'backend': 'localmongodb', + 'host': DATABASE_HOST, + 'port': DATABASE_PORT, + 'name': DATABASE_NAME, + 'connection_timeout': 5000, + 'max_tries': 3, + 'replicaset': None, + 'ssl': False, + 'login': None, + 'password': None, + 'ca_cert': 'ca_cert', + 'certfile': 'certfile', + 'keyfile': 'keyfile', + 'keyfile_passphrase': 'passphrase', + 'crlfile': 'crlfile', + } + + assert bigchaindb.config == { + 'CONFIGURED': True, + 'server': { + 'bind': SERVER_BIND, + 'loglevel': 'info', + 'workers': None, + }, + 'wsserver': { + 'scheme': WSSERVER_SCHEME, + 'host': WSSERVER_HOST, + 'port': WSSERVER_PORT, + 'advertised_scheme': WSSERVER_ADVERTISED_SCHEME, + 'advertised_host': WSSERVER_ADVERTISED_HOST, + 'advertised_port': WSSERVER_ADVERTISED_PORT, + }, + 'database': database_mongodb, + 'tendermint': { + 'host': 'localhost', + 'port': 26657, + 'version': 'v0.31.5' + }, + 'log': { + 'file': LOG_FILE, + 'level_console': 'debug', + 'error_file': log_config['handlers']['errors']['filename'], + 'level_console': 'debug', + 'level_logfile': 'info', + 'datefmt_console': log_config['formatters']['console']['datefmt'], + 'datefmt_logfile': log_config['formatters']['file']['datefmt'], + 'fmt_console': log_config['formatters']['console']['format'], + 'fmt_logfile': log_config['formatters']['file']['format'], + 'granular_levels': {}, + }, + } + + +def test_autoconfigure_env_precedence(monkeypatch): + file_config = { + 'database': {'host': 'test-host', 'name': 'bigchaindb', 'port': 28015} + } + monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) + monkeypatch.setattr('os.environ', {'PLANETMINT_DATABASE_NAME': 'test-dbname', + 'PLANETMINT_DATABASE_PORT': '4242', + 'PLANETMINT_SERVER_BIND': 'localhost:9985'}) + + import bigchaindb + from bigchaindb import config_utils + config_utils.autoconfigure() + + assert bigchaindb.config['CONFIGURED'] + assert bigchaindb.config['database']['host'] == 'test-host' + assert bigchaindb.config['database']['name'] == 'test-dbname' + assert bigchaindb.config['database']['port'] == 4242 + assert bigchaindb.config['server']['bind'] == 'localhost:9985' + + +def test_autoconfigure_explicit_file(monkeypatch): + from bigchaindb import config_utils + + def file_config(*args, **kwargs): + raise FileNotFoundError() + + monkeypatch.setattr('bigchaindb.config_utils.file_config', file_config) + + with pytest.raises(FileNotFoundError): + config_utils.autoconfigure(filename='autoexec.bat') + + +def test_update_config(monkeypatch): + import bigchaindb + from bigchaindb import config_utils + + file_config = { + 'database': {'host': 'test-host', 'name': 'bigchaindb', 'port': 28015} + } + monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) + config_utils.autoconfigure(config=file_config) + + # update configuration, retaining previous changes + config_utils.update_config({'database': {'port': 28016, 'name': 'bigchaindb_other'}}) + + assert bigchaindb.config['database']['host'] == 'test-host' + assert bigchaindb.config['database']['name'] == 'bigchaindb_other' + assert bigchaindb.config['database']['port'] == 28016 + + +def test_file_config(): + from bigchaindb.config_utils import file_config, CONFIG_DEFAULT_PATH + with patch('builtins.open', mock_open(read_data='{}')) as m: + config = file_config() + m.assert_called_once_with(CONFIG_DEFAULT_PATH) + assert config == {} + + +def test_invalid_file_config(): + from bigchaindb.config_utils import file_config + from bigchaindb.common import exceptions + with patch('builtins.open', mock_open(read_data='{_INVALID_JSON_}')): + with pytest.raises(exceptions.ConfigurationError): + file_config() + + +def test_write_config(): + from bigchaindb.config_utils import write_config, CONFIG_DEFAULT_PATH + m = mock_open() + with patch('builtins.open', m): + write_config({}) + m.assert_called_once_with(CONFIG_DEFAULT_PATH, 'w') + handle = m() + handle.write.assert_called_once_with('{}') + + +@pytest.mark.parametrize('env_name,env_value,config_key', ( + ('PLANETMINT_DATABASE_BACKEND', 'test-backend', 'backend'), + ('PLANETMINT_DATABASE_HOST', 'test-host', 'host'), + ('PLANETMINT_DATABASE_PORT', 4242, 'port'), + ('PLANETMINT_DATABASE_NAME', 'test-db', 'name'), +)) +def test_database_envs(env_name, env_value, config_key, monkeypatch): + import bigchaindb + + monkeypatch.setattr('os.environ', {env_name: env_value}) + bigchaindb.config_utils.autoconfigure() + + expected_config = copy.deepcopy(bigchaindb.config) + expected_config['database'][config_key] = env_value + + assert bigchaindb.config == expected_config diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..04a68f0 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,118 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + + +@pytest.fixture +def config(request, monkeypatch): + backend = request.config.getoption('--database-backend') + if backend == 'mongodb-ssl': + backend = 'mongodb' + + config = { + 'database': { + 'backend': backend, + 'host': 'host', + 'port': 28015, + 'name': 'bigchain', + 'replicaset': 'bigchain-rs', + 'connection_timeout': 5000, + 'max_tries': 3 + }, + 'tendermint': { + 'host': 'localhost', + 'port': 26657, + }, + 'CONFIGURED': True, + } + + monkeypatch.setattr('bigchaindb.config', config) + + return config + + +def test_bigchain_class_default_initialization(config): + from bigchaindb import BigchainDB + from bigchaindb.validation import BaseValidationRules + from bigchaindb.backend.connection import Connection + bigchain = BigchainDB) + assert isinstance(bigchain.connection, Connection) + assert bigchain.connection.host == config['database']['host'] + assert bigchain.connection.port == config['database']['port'] + assert bigchain.connection.dbname == config['database']['name'] + assert bigchain.validation == BaseValidationRules + + +def test_bigchain_class_initialization_with_parameters(): + from bigchaindb import BigchainDB + from bigchaindb.backend import connect + from bigchaindb.validation import BaseValidationRules + init_db_kwargs = { + 'backend': 'localmongodb', + 'host': 'this_is_the_db_host', + 'port': 12345, + 'name': 'this_is_the_db_name', + } + connection = connect(**init_db_kwargs) + bigchain = BigchainDBconnection=connection) + assert bigchain.connection == connection + assert bigchain.connection.host == init_db_kwargs['host'] + assert bigchain.connection.port == init_db_kwargs['port'] + assert bigchain.connection.dbname == init_db_kwargs['name'] + assert bigchain.validation == BaseValidationRules + + +@pytest.mark.bdb +def test_get_spent_issue_1271(b, alice, bob, carol): + from bigchaindb.models import Transaction + + tx_1 = Transaction.create( + [carol.public_key], + [([carol.public_key], 8)], + ).sign([carol.private_key]) + assert tx_1.validate(b) + b.store_bulk_transactions([tx_1]) + + tx_2 = Transaction.transfer( + tx_1.to_inputs(), + [([bob.public_key], 2), + ([alice.public_key], 2), + ([carol.public_key], 4)], + asset_id=tx_1.id, + ).sign([carol.private_key]) + assert tx_2.validate(b) + b.store_bulk_transactions([tx_2]) + + tx_3 = Transaction.transfer( + tx_2.to_inputs()[2:3], + [([alice.public_key], 1), + ([carol.public_key], 3)], + asset_id=tx_1.id, + ).sign([carol.private_key]) + assert tx_3.validate(b) + b.store_bulk_transactions([tx_3]) + + tx_4 = Transaction.transfer( + tx_2.to_inputs()[1:2] + tx_3.to_inputs()[0:1], + [([bob.public_key], 3)], + asset_id=tx_1.id, + ).sign([alice.private_key]) + assert tx_4.validate(b) + b.store_bulk_transactions([tx_4]) + + tx_5 = Transaction.transfer( + tx_2.to_inputs()[0:1], + [([alice.public_key], 2)], + asset_id=tx_1.id, + ).sign([bob.private_key]) + assert tx_5.validate(b) + + b.store_bulk_transactions([tx_5]) + + assert b.get_spent(tx_2.id, 0) == tx_5 + assert not b.get_spent(tx_5.id, 0) + assert b.get_outputs_filtered(alice.public_key) + assert b.get_outputs_filtered(alice.public_key, spent=False) diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..5ba8434 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,14 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +import subprocess + + +def test_build_root_docs(): + proc = subprocess.Popen(['bash'], stdin=subprocess.PIPE) + proc.stdin.write('cd docs/root; make html'.encode()) + proc.stdin.close() + assert proc.wait() == 0 diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 0000000..663e08d --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,75 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + + +def test_event_handler(): + from bigchaindb.events import EventTypes, Event, Exchange + + # create and event + event_data = {'msg': 'some data'} + event = Event(EventTypes.BLOCK_VALID, event_data) + + # create the events pub sub + exchange = Exchange() + + sub0 = exchange.get_subscriber_queue(EventTypes.BLOCK_VALID) + sub1 = exchange.get_subscriber_queue(EventTypes.BLOCK_VALID | + EventTypes.BLOCK_INVALID) + # Subscribe to all events + sub2 = exchange.get_subscriber_queue() + sub3 = exchange.get_subscriber_queue(EventTypes.BLOCK_INVALID) + + # push and event to the queue + exchange.dispatch(event) + + # get the event from the queue + event_sub0 = sub0.get() + event_sub1 = sub1.get() + event_sub2 = sub2.get() + + assert event_sub0.type == event.type + assert event_sub0.data == event.data + + assert event_sub1.type == event.type + assert event_sub1.data == event.data + + assert event_sub2.type == event.type + assert event_sub2.data == event.data + + assert sub3.qsize() == 0 + + +def test_event_handler_raises_when_called_after_start(): + from bigchaindb.events import Exchange, POISON_PILL + + exchange = Exchange() + publisher_queue = exchange.get_publisher_queue() + publisher_queue.put(POISON_PILL) + exchange.run() + + with pytest.raises(RuntimeError): + exchange.get_subscriber_queue() + + +def test_exchange_stops_with_poison_pill(): + from bigchaindb.events import EventTypes, Event, Exchange, POISON_PILL + + # create and event + event_data = {'msg': 'some data'} + event = Event(EventTypes.BLOCK_VALID, event_data) + + # create the events pub sub + exchange = Exchange() + + publisher_queue = exchange.get_publisher_queue() + + # push and event to the queue + publisher_queue.put(event) + publisher_queue.put(POISON_PILL) + exchange.run() + + assert publisher_queue.qsize() == 0 diff --git a/tests/test_parallel_validation.py b/tests/test_parallel_validation.py new file mode 100644 index 0000000..3d21553 --- /dev/null +++ b/tests/test_parallel_validation.py @@ -0,0 +1,134 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + +from bigchaindb.common.crypto import generate_key_pair +from bigchaindb.models import Transaction + + +pytestmark = pytest.mark.tendermint + + +def generate_create_and_transfer(keypair=None): + if not keypair: + keypair = generate_key_pair() + priv_key, pub_key = keypair + create_tx = Transaction.create([pub_key], [([pub_key], 10)]).sign([priv_key]) + transfer_tx = Transaction.transfer( + create_tx.to_inputs(), + [([pub_key], 10)], + asset_id=create_tx.id).sign([priv_key]) + return create_tx, transfer_tx + + +def test_validation_worker_process_multiple_transactions(b): + import multiprocessing as mp + from bigchaindb.parallel_validation import ValidationWorker, RESET, EXIT + + keypair = generate_key_pair() + create_tx, transfer_tx = generate_create_and_transfer(keypair) + double_spend = Transaction.transfer( + create_tx.to_inputs(), + [([keypair.public_key], 10)], + asset_id=create_tx.id).sign([keypair.private_key]) + + in_queue, results_queue = mp.Queue(), mp.Queue() + vw = ValidationWorker(in_queue, results_queue) + + # Note: in the following instructions, the worker will encounter two + # `RESET` messages, and an `EXIT` message. When a worker processes a + # `RESET` message, it forgets all transactions it has validated. This allow + # us to re-validate the same transactions. This won't happen in real life, + # but it's quite handy to check if the worker actually forgot about the + # past transactions (if not, it will return `False` because the + # transactions look like a double spend). + # `EXIT` makes the worker to stop the infinite loop. + in_queue.put((0, create_tx.to_dict())) + in_queue.put((10, transfer_tx.to_dict())) + in_queue.put((20, double_spend.to_dict())) + in_queue.put(RESET) + in_queue.put((0, create_tx.to_dict())) + in_queue.put((5, transfer_tx.to_dict())) + in_queue.put(RESET) + in_queue.put((20, create_tx.to_dict())) + in_queue.put((25, double_spend.to_dict())) + in_queue.put((30, transfer_tx.to_dict())) + in_queue.put(EXIT) + + vw.run() + + assert results_queue.get() == (0, create_tx) + assert results_queue.get() == (10, transfer_tx) + assert results_queue.get() == (20, False) + assert results_queue.get() == (0, create_tx) + assert results_queue.get() == (5, transfer_tx) + assert results_queue.get() == (20, create_tx) + assert results_queue.get() == (25, double_spend) + assert results_queue.get() == (30, False) + + +def test_parallel_validator_routes_transactions_correctly(b, monkeypatch): + import os + from collections import defaultdict + import multiprocessing as mp + from json import dumps + from bigchaindb.parallel_validation import ParallelValidator + + # We want to make sure that the load is distributed across all workers. + # Since introspection on an object running on a different process is + # difficult, we create an additional queue where every worker can emit its + # PID every time validation is called. + validation_called_by = mp.Queue() + + # Validate is now a passthrough, and every time it is called it will emit + # the PID of its worker to the designated queue. + def validate(self, dict_transaction): + validation_called_by.put((os.getpid(), dict_transaction['id'])) + return dict_transaction + + monkeypatch.setattr( + 'bigchaindb.parallel_validation.ValidationWorker.validate', + validate) + + # Transaction routing uses the `id` of the transaction. This test strips + # down a transaction to just its `id`. We have two workers, so even ids + # will be processed by one worker, odd ids by the other. + transactions = [{'id': '0'}, {'id': '1'}, {'id': '2'}, {'id': '3'}] + + pv = ParallelValidator(number_of_workers=2) + pv.start() + + # ParallelValidator is instantiated once, and then used several times. + # Here we simulate this scenario by running it an arbitrary number of + # times. + # Note that the `ParallelValidator.result` call resets the object, and + # makes it ready to validate a new set of transactions. + for _ in range(2): + # First, we push the transactions to the parallel validator instance + for transaction in transactions: + pv.validate(dumps(transaction).encode('utf8')) + + assert pv.result(timeout=1) == transactions + + # Now we analize the transaction processed by the workers + worker_to_transactions = defaultdict(list) + for _ in transactions: + worker_pid, transaction_id = validation_called_by.get() + worker_to_transactions[worker_pid].append(transaction_id) + + # The transactions are stored in two buckets. + for _, transaction_ids in worker_to_transactions.items(): + assert len(transaction_ids) == 2 + + # We have two workers, hence we have two different routes for + # transactions. We have the route for even transactions, and the + # route for odd transactions. Since we don't know which worker + # processed what, we test that the transactions processed by a + # worker are all even or all odd. + assert (all(filter(lambda x: int(x) % 2 == 0, transaction_ids)) or + all(filter(lambda x: int(x) % 2 == 1, transaction_ids))) + + pv.stop() diff --git a/tests/test_txlist.py b/tests/test_txlist.py new file mode 100644 index 0000000..c5d930c --- /dev/null +++ b/tests/test_txlist.py @@ -0,0 +1,48 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""Test getting a list of transactions from the backend. + +This test module defines it's own fixture which is used by all the tests. +""" +import pytest + + +@pytest.fixture +def txlist(b, user_pk, user2_pk, user_sk, user2_sk): + from bigchaindb.models import Transaction + + # Create two CREATE transactions + create1 = Transaction.create([user_pk], [([user2_pk], 6)]) \ + .sign([user_sk]) + + create2 = Transaction.create([user2_pk], + [([user2_pk], 5), ([user_pk], 5)]) \ + .sign([user2_sk]) + + # Create a TRANSFER transactions + transfer1 = Transaction.transfer(create1.to_inputs(), + [([user_pk], 8)], + create1.id).sign([user2_sk]) + + b.store_bulk_transactions([create1, create2, transfer1]) + + return type('', (), { + 'create1': create1, + 'transfer1': transfer1, + }) + + +@pytest.mark.bdb +def test_get_txlist_by_asset(b, txlist): + res = b.get_transactions_filtered(txlist.create1.id) + assert sorted(set(tx.id for tx in res)) == sorted( + set([txlist.transfer1.id, txlist.create1.id])) + + +@pytest.mark.bdb +def test_get_txlist_by_operation(b, txlist): + res = b.get_transactions_filtered(txlist.create1.id, operation='CREATE') + assert set(tx.id for tx in res) == {txlist.create1.id} diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..50ade81 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,173 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import queue +from unittest.mock import patch, call + +import pytest + + +@pytest.fixture +def mock_queue(monkeypatch): + + class MockQueue: + items = [] + + def get(self, timeout=None): + try: + return self.items.pop() + except IndexError: + if timeout: + raise queue.Empty() + raise + + def put(self, item): + self.items.append(item) + + mockqueue = MockQueue() + + monkeypatch.setattr('queue.Queue', lambda: mockqueue) + return mockqueue + + +def test_empty_pool_is_populated_with_instances(mock_queue): + from bigchaindb import utils + + pool = utils.pool(lambda: 'hello', 4) + + assert len(mock_queue.items) == 0 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 1 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 2 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 3 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 4 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 4 + + +def test_pool_blocks_if_no_instances_available(mock_queue): + from bigchaindb import utils + + pool = utils.pool(lambda: 'hello', 4) + + assert len(mock_queue.items) == 0 + + # We need to manually trigger the `__enter__` method so the context + # manager will "hang" and not return the resource to the pool + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + # We need to keep a reference of the last context manager so we can + # manually release the resource + last = pool() + assert last.__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + # This would block using `queue.Queue` but since we mocked it it will + # just raise a IndexError because it's trying to pop from an empty list. + with pytest.raises(IndexError): + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + # Release the last resource + last.__exit__(None, None, None) + assert len(mock_queue.items) == 1 + + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + +def test_pool_raises_empty_exception_when_timeout(mock_queue): + from bigchaindb import utils + + pool = utils.pool(lambda: 'hello', 1, timeout=1) + + assert len(mock_queue.items) == 0 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 1 + + # take the only resource available + assert pool().__enter__() == 'hello' + + with pytest.raises(queue.Empty): + with pool() as instance: + assert instance == 'hello' + + +@patch('multiprocessing.Process') +def test_process_group_instantiates_and_start_processes(mock_process): + from bigchaindb.utils import ProcessGroup + + def noop(): + pass + + concurrency = 10 + + pg = ProcessGroup(concurrency=concurrency, group='test_group', target=noop) + pg.start() + + mock_process.assert_has_calls([call(group='test_group', target=noop, + name=None, args=(), kwargs={}, + daemon=None) + for i in range(concurrency)], any_order=True) + + for process in pg.processes: + process.start.assert_called_with() + + +def test_lazy_execution(): + from bigchaindb.utils import Lazy + + lz = Lazy() + lz.split(',')[1].split(' ').pop(1).strip() + result = lz.run('Like humans, cats tend to favor one paw over another') + assert result == 'cats' + + class Cat: + def __init__(self, name): + self.name = name + + cat = Cat('Shmui') + + lz = Lazy() + lz.name.upper() + result = lz.run(cat) + assert result == 'SHMUI' + + +def test_process_set_title(): + from uuid import uuid4 + from multiprocessing import Queue + from setproctitle import getproctitle + from bigchaindb.utils import Process + + queue = Queue() + uuid = str(uuid4()) + + process = Process(target=lambda: queue.put(getproctitle()), + name=uuid) + process.start() + assert queue.get() == uuid diff --git a/tests/upsert_validator/__init__.py b/tests/upsert_validator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/upsert_validator/conftest.py b/tests/upsert_validator/conftest.py new file mode 100644 index 0000000..08ebad5 --- /dev/null +++ b/tests/upsert_validator/conftest.py @@ -0,0 +1,46 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 +from unittest.mock import patch + +import pytest + +from bigchaindb.backend.localmongodb import query +from bigchaindb.upsert_validator import ValidatorElection + + +@pytest.fixture +def valid_upsert_validator_election_b(b, node_key, new_validator): + voters = ValidatorElection.recipients(b) + return ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +@pytest.fixture +@patch('bigchaindb.elections.election.uuid4', lambda: 'mock_uuid4') +def fixed_seed_election(b_mock, node_key, new_validator): + voters = ValidatorElection.recipients(b_mock) + return ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +@pytest.fixture +def concluded_election(b, ongoing_validator_election, ed25519_node_keys): + query.store_election(b.connection, ongoing_validator_election.id, + 2, is_concluded=True) + return ongoing_validator_election + + +@pytest.fixture +def inconclusive_election(b, ongoing_validator_election, new_validator): + validators = b.get_validators(height=1) + validators[0]['voting_power'] = 15 + validator_update = {'validators': validators, + 'height': 2, + 'election_id': 'some_other_election'} + + query.store_validator_set(b.connection, validator_update) + return ongoing_validator_election diff --git a/tests/upsert_validator/test_upsert_validator_vote.py b/tests/upsert_validator/test_upsert_validator_vote.py new file mode 100644 index 0000000..dd107a7 --- /dev/null +++ b/tests/upsert_validator/test_upsert_validator_vote.py @@ -0,0 +1,340 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest +import codecs + +from bigchaindb.elections.election import Election +from bigchaindb.tendermint_utils import public_key_to_base64 +from bigchaindb.upsert_validator import ValidatorElection +from bigchaindb.common.exceptions import AmountError +from bigchaindb.common.crypto import generate_key_pair +from bigchaindb.common.exceptions import ValidationError +from bigchaindb.common.transaction_mode_types import BROADCAST_TX_COMMIT +from bigchaindb.elections.vote import Vote +from tests.utils import generate_block, gen_vote + +pytestmark = [pytest.mark.execute] + + +@pytest.mark.bdb +def test_upsert_validator_valid_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) + + vote = Vote.generate([input0], + [([election_pub_key], votes)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + assert vote.validate(b_mock) + + +@pytest.mark.bdb +def test_upsert_validator_valid_non_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) + + # Ensure that threshold conditions are now allowed + with pytest.raises(ValidationError): + Vote.generate([input0], + [([election_pub_key, key0.public_key], votes)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + + +@pytest.mark.bdb +def test_upsert_validator_delegate_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + alice = generate_key_pair() + + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + + delegate_vote = Vote.generate([input0], + [([alice.public_key], 3), ([key0.public_key], votes-3)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + + assert delegate_vote.validate(b_mock) + + b_mock.store_bulk_transactions([delegate_vote]) + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) + + alice_votes = delegate_vote.to_inputs()[0] + alice_casted_vote = Vote.generate([alice_votes], + [([election_pub_key], 3)], + election_id=valid_upsert_validator_election.id)\ + .sign([alice.private_key]) + assert alice_casted_vote.validate(b_mock) + + key0_votes = delegate_vote.to_inputs()[1] + key0_casted_vote = Vote.generate([key0_votes], + [([election_pub_key], votes-3)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + assert key0_casted_vote.validate(b_mock) + + +@pytest.mark.bdb +def test_upsert_validator_invalid_election_vote(b_mock, valid_upsert_validator_election, ed25519_node_keys): + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + + election_pub_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) + + vote = Vote.generate([input0], + [([election_pub_key], votes+1)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + + with pytest.raises(AmountError): + assert vote.validate(b_mock) + + +@pytest.mark.bdb +def test_valid_election_votes_received(b_mock, valid_upsert_validator_election, ed25519_node_keys): + alice = generate_key_pair() + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + assert valid_upsert_validator_election.get_commited_votes(b_mock) == 0 + + input0 = valid_upsert_validator_election.to_inputs()[0] + votes = valid_upsert_validator_election.outputs[0].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + + # delegate some votes to alice + delegate_vote = Vote.generate([input0], + [([alice.public_key], 4), ([key0.public_key], votes-4)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + b_mock.store_bulk_transactions([delegate_vote]) + assert valid_upsert_validator_election.get_commited_votes(b_mock) == 0 + + election_public_key = ValidatorElection.to_public_key(valid_upsert_validator_election.id) + alice_votes = delegate_vote.to_inputs()[0] + key0_votes = delegate_vote.to_inputs()[1] + + alice_casted_vote = Vote.generate([alice_votes], + [([election_public_key], 2), ([alice.public_key], 2)], + election_id=valid_upsert_validator_election.id)\ + .sign([alice.private_key]) + + assert alice_casted_vote.validate(b_mock) + b_mock.store_bulk_transactions([alice_casted_vote]) + + # Check if the delegated vote is count as valid vote + assert valid_upsert_validator_election.get_commited_votes(b_mock) == 2 + + key0_casted_vote = Vote.generate([key0_votes], + [([election_public_key], votes-4)], + election_id=valid_upsert_validator_election.id)\ + .sign([key0.private_key]) + + assert key0_casted_vote.validate(b_mock) + b_mock.store_bulk_transactions([key0_casted_vote]) + + assert valid_upsert_validator_election.get_commited_votes(b_mock) == votes - 2 + + +@pytest.mark.bdb +def test_valid_election_conclude(b_mock, valid_upsert_validator_election, ed25519_node_keys): + + # Node 0: cast vote + tx_vote0 = gen_vote(valid_upsert_validator_election, 0, ed25519_node_keys) + + # check if the vote is valid even before the election doesn't exist + with pytest.raises(ValidationError): + assert tx_vote0.validate(b_mock) + + # store election + b_mock.store_bulk_transactions([valid_upsert_validator_election]) + # cannot conclude election as not votes exist + assert not valid_upsert_validator_election.has_concluded(b_mock) + + # validate vote + assert tx_vote0.validate(b_mock) + assert not valid_upsert_validator_election.has_concluded(b_mock, [tx_vote0]) + + b_mock.store_bulk_transactions([tx_vote0]) + assert not valid_upsert_validator_election.has_concluded(b_mock) + + # Node 1: cast vote + tx_vote1 = gen_vote(valid_upsert_validator_election, 1, ed25519_node_keys) + + # Node 2: cast vote + tx_vote2 = gen_vote(valid_upsert_validator_election, 2, ed25519_node_keys) + + # Node 3: cast vote + tx_vote3 = gen_vote(valid_upsert_validator_election, 3, ed25519_node_keys) + + assert tx_vote1.validate(b_mock) + assert not valid_upsert_validator_election.has_concluded(b_mock, [tx_vote1]) + + # 2/3 is achieved in the same block so the election can be.has_concludedd + assert valid_upsert_validator_election.has_concluded(b_mock, [tx_vote1, tx_vote2]) + + b_mock.store_bulk_transactions([tx_vote1]) + assert not valid_upsert_validator_election.has_concluded(b_mock) + + assert tx_vote2.validate(b_mock) + assert tx_vote3.validate(b_mock) + + # conclusion can be triggered my different votes in the same block + assert valid_upsert_validator_election.has_concluded(b_mock, [tx_vote2]) + assert valid_upsert_validator_election.has_concluded(b_mock, [tx_vote2, tx_vote3]) + + b_mock.store_bulk_transactions([tx_vote2]) + + # Once the blockchain records >2/3 of the votes the election is assumed to be.has_concludedd + # so any invocation of `.has_concluded` for that election should return False + assert not valid_upsert_validator_election.has_concluded(b_mock) + + # Vote is still valid but the election cannot be.has_concludedd as it it assmed that it has + # been.has_concludedd before + assert tx_vote3.validate(b_mock) + assert not valid_upsert_validator_election.has_concluded(b_mock, [tx_vote3]) + + +@pytest.mark.abci +def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys): + + if b.get_latest_block()['height'] == 0: + generate_block(b) + + (node_pub, _) = list(node_keys.items())[0] + + validators = [{'public_key': {'type': 'ed25519-base64', 'value': node_pub}, + 'voting_power': 10}] + + latest_block = b.get_latest_block() + # reset the validator set + b.store_validator_set(latest_block['height'], validators) + generate_block(b) + + power = 1 + public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403' + public_key64 = public_key_to_base64(public_key) + new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'}, + 'node_id': 'some_node_id', + 'power': power} + + voters = ValidatorElection.recipients(b) + election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + code, message = b.write_transaction(election, BROADCAST_TX_COMMIT) + assert code == 202 + assert b.get_transaction(election.id) + + tx_vote = gen_vote(election, 0, ed25519_node_keys) + assert tx_vote.validate(b) + code, message = b.write_transaction(tx_vote, BROADCAST_TX_COMMIT) + assert code == 202 + + resp = b.get_validators() + validator_pub_keys = [] + for v in resp: + validator_pub_keys.append(v['public_key']['value']) + + assert (public_key64 in validator_pub_keys) + new_validator_set = b.get_validators() + validator_pub_keys = [] + for v in new_validator_set: + validator_pub_keys.append(v['public_key']['value']) + + assert (public_key64 in validator_pub_keys) + + +@pytest.mark.bdb +def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): + reset_validator_set(b, node_keys, 1) + + power = 1 + public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403' + public_key64 = public_key_to_base64(public_key) + new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'}, + 'node_id': 'some_node_id', + 'power': power} + voters = ValidatorElection.recipients(b) + election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator).sign([node_key.private_key]) + # store election + b.store_bulk_transactions([election]) + + tx_vote0 = gen_vote(election, 0, ed25519_node_keys) + tx_vote1 = gen_vote(election, 1, ed25519_node_keys) + tx_vote2 = gen_vote(election, 2, ed25519_node_keys) + + assert not election.has_concluded(b, [tx_vote0]) + assert not election.has_concluded(b, [tx_vote0, tx_vote1]) + assert election.has_concluded(b, [tx_vote0, tx_vote1, tx_vote2]) + + assert Election.process_block(b, 4, [tx_vote0]) == [] + assert Election.process_block(b, 4, [tx_vote0, tx_vote1]) == [] + + update = Election.process_block(b, 4, [tx_vote0, tx_vote1, tx_vote2]) + assert len(update) == 1 + update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') + assert update_public_key == public_key64 + + # remove validator + power = 0 + new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'}, + 'node_id': 'some_node_id', + 'power': power} + voters = ValidatorElection.recipients(b) + election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator).sign([node_key.private_key]) + # store election + b.store_bulk_transactions([election]) + + tx_vote0 = gen_vote(election, 0, ed25519_node_keys) + tx_vote1 = gen_vote(election, 1, ed25519_node_keys) + tx_vote2 = gen_vote(election, 2, ed25519_node_keys) + + b.store_bulk_transactions([tx_vote0, tx_vote1]) + + update = Election.process_block(b, 9, [tx_vote2]) + assert len(update) == 1 + update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n') + assert update_public_key == public_key64 + + # assert that the public key is not a part of the current validator set + for v in b.get_validators(10): + assert not v['public_key']['value'] == public_key64 + + +# ============================================================================ +# Helper functions +# ============================================================================ + +def reset_validator_set(b, node_keys, height): + validators = [] + for (node_pub, _) in node_keys.items(): + validators.append({'public_key': {'type': 'ed25519-base64', + 'value': node_pub}, + 'voting_power': 10}) + b.store_validator_set(height, validators) diff --git a/tests/upsert_validator/test_validator_election.py b/tests/upsert_validator/test_validator_election.py new file mode 100644 index 0000000..eff7a2d --- /dev/null +++ b/tests/upsert_validator/test_validator_election.py @@ -0,0 +1,180 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 +from argparse import Namespace +from unittest.mock import patch + +import pytest + +from bigchaindb.tendermint_utils import public_key_to_base64 +from bigchaindb.upsert_validator import ValidatorElection +from bigchaindb.common.exceptions import (DuplicateTransaction, + UnequalValidatorSet, + InvalidProposer, + MultipleInputsError, + InvalidPowerChange) + +pytestmark = pytest.mark.bdb + + +def test_upsert_validator_valid_election(b_mock, new_validator, node_key): + voters = ValidatorElection.recipients(b_mock) + election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + assert election.validate(b_mock) + + +def test_upsert_validator_invalid_election_public_key(b_mock, new_validator, node_key): + from bigchaindb.common.exceptions import InvalidPublicKey + + for iv in ['ed25519-base32', 'ed25519-base64']: + new_validator['public_key']['type'] = iv + voters = ValidatorElection.recipients(b_mock) + + with pytest.raises(InvalidPublicKey): + ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + +def test_upsert_validator_invalid_power_election(b_mock, new_validator, node_key): + voters = ValidatorElection.recipients(b_mock) + new_validator['power'] = 30 + + election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + with pytest.raises(InvalidPowerChange): + election.validate(b_mock) + + +def test_upsert_validator_invalid_proposed_election(b_mock, new_validator, node_key): + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + voters = ValidatorElection.recipients(b_mock) + election = ValidatorElection.generate([alice.public_key], + voters, + new_validator, None).sign([alice.private_key]) + with pytest.raises(InvalidProposer): + election.validate(b_mock) + + +def test_upsert_validator_invalid_inputs_election(b_mock, new_validator, node_key): + from bigchaindb.common.crypto import generate_key_pair + + alice = generate_key_pair() + voters = ValidatorElection.recipients(b_mock) + election = ValidatorElection.generate([node_key.public_key, alice.public_key], + voters, + new_validator, None).sign([node_key.private_key, alice.private_key]) + with pytest.raises(MultipleInputsError): + election.validate(b_mock) + + +@patch('bigchaindb.elections.election.uuid4', lambda: 'mock_uuid4') +def test_upsert_validator_invalid_election(b_mock, new_validator, node_key, fixed_seed_election): + voters = ValidatorElection.recipients(b_mock) + duplicate_election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + + with pytest.raises(DuplicateTransaction): + fixed_seed_election.validate(b_mock, [duplicate_election]) + + b_mock.store_bulk_transactions([fixed_seed_election]) + + with pytest.raises(DuplicateTransaction): + duplicate_election.validate(b_mock) + + # Try creating an election with incomplete voter set + invalid_election = ValidatorElection.generate([node_key.public_key], + voters[1:], + new_validator, None).sign([node_key.private_key]) + + with pytest.raises(UnequalValidatorSet): + invalid_election.validate(b_mock) + + recipients = ValidatorElection.recipients(b_mock) + altered_recipients = [] + for r in recipients: + ([r_public_key], voting_power) = r + altered_recipients.append(([r_public_key], voting_power - 1)) + + # Create a transaction which doesn't enfore the network power + tx_election = ValidatorElection.generate([node_key.public_key], + altered_recipients, + new_validator, None).sign([node_key.private_key]) + + with pytest.raises(UnequalValidatorSet): + tx_election.validate(b_mock) + + +def test_get_status_ongoing(b, ongoing_validator_election, new_validator): + status = ValidatorElection.ONGOING + resp = ongoing_validator_election.get_status(b) + assert resp == status + + +def test_get_status_concluded(b, concluded_election, new_validator): + status = ValidatorElection.CONCLUDED + resp = concluded_election.get_status(b) + assert resp == status + + +def test_get_status_inconclusive(b, inconclusive_election, new_validator): + def set_block_height_to_3(): + return {'height': 3} + + def custom_mock_get_validators(height): + if height >= 3: + return [{'pub_key': {'data': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=', + 'type': 'AC26791624DE60'}, + 'voting_power': 15}, + {'pub_key': {'data': 'GIijU7GBcVyiVUcB0GwWZbxCxdk2xV6pxdvL24s/AqM=', + 'type': 'AC26791624DE60'}, + 'voting_power': 7}, + {'pub_key': {'data': 'JbfwrLvCVIwOPm8tj8936ki7IYbmGHjPiKb6nAZegRA=', + 'type': 'AC26791624DE60'}, + 'voting_power': 10}, + {'pub_key': {'data': 'PecJ58SaNRsWJZodDmqjpCWqG6btdwXFHLyE40RYlYM=', + 'type': 'AC26791624DE60'}, + 'voting_power': 8}] + else: + return [{'pub_key': {'data': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=', + 'type': 'AC26791624DE60'}, + 'voting_power': 9}, + {'pub_key': {'data': 'GIijU7GBcVyiVUcB0GwWZbxCxdk2xV6pxdvL24s/AqM=', + 'type': 'AC26791624DE60'}, + 'voting_power': 7}, + {'pub_key': {'data': 'JbfwrLvCVIwOPm8tj8936ki7IYbmGHjPiKb6nAZegRA=', + 'type': 'AC26791624DE60'}, + 'voting_power': 10}, + {'pub_key': {'data': 'PecJ58SaNRsWJZodDmqjpCWqG6btdwXFHLyE40RYlYM=', + 'type': 'AC26791624DE60'}, + 'voting_power': 8}] + + b.get_validators = custom_mock_get_validators + b.get_latest_block = set_block_height_to_3 + status = ValidatorElection.INCONCLUSIVE + resp = inconclusive_election.get_status(b) + assert resp == status + + +def test_upsert_validator_show(caplog, ongoing_validator_election, b): + from bigchaindb.commands.bigchaindb import run_election_show + + election_id = ongoing_validator_election.id + public_key = public_key_to_base64(ongoing_validator_election.asset['data']['public_key']['value']) + power = ongoing_validator_election.asset['data']['power'] + node_id = ongoing_validator_election.asset['data']['node_id'] + status = ValidatorElection.ONGOING + + show_args = Namespace(action='show', + election_id=election_id) + + msg = run_election_show(show_args, b) + + assert msg == f'public_key={public_key}\npower={power}\nnode_id={node_id}\nstatus={status}' diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..fc74f6c --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,114 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import base58 +import base64 +import random + +from functools import singledispatch + +from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection +from bigchaindb.backend.schema import TABLES +from bigchaindb.common import crypto +from bigchaindb.common.transaction_mode_types import BROADCAST_TX_COMMIT +from bigchaindb.elections.election import Election, Vote +from bigchaindb.tendermint_utils import key_to_base64 + + +@singledispatch +def flush_db(connection, dbname): + raise NotImplementedError + + +@flush_db.register(LocalMongoDBConnection) +def flush_localmongo_db(connection, dbname): + for t in TABLES: + getattr(connection.conn[dbname], t).delete_many({}) + + +def generate_block(bigchain): + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.models import Transaction + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None)\ + .sign([alice.private_key]) + + code, message = bigchain.write_transaction(tx, BROADCAST_TX_COMMIT) + assert code == 202 + + +def to_inputs(election, i, ed25519_node_keys): + input0 = election.to_inputs()[i] + votes = election.outputs[i].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + return (input0, votes, key0) + + +def gen_vote(election, i, ed25519_node_keys): + (input_i, votes_i, key_i) = to_inputs(election, i, ed25519_node_keys) + election_pub_key = Election.to_public_key(election.id) + return Vote.generate([input_i], + [([election_pub_key], votes_i)], + election_id=election.id)\ + .sign([key_i.private_key]) + + +def generate_validators(powers): + """Generates an arbitrary number of validators with random public keys. + + The object under the `storage` key is in the format expected by DB. + + The object under the `eleciton` key is in the format expected by + the upsert validator election. + + `public_key`, `private_key` are in the format used for signing transactions. + + Args: + powers: A list of intergers representing the voting power to + assign to the corresponding validators. + """ + validators = [] + for power in powers: + kp = crypto.generate_key_pair() + validators.append({ + 'storage': { + 'public_key': { + 'value': key_to_base64(base58.b58decode(kp.public_key).hex()), + 'type': 'ed25519-base64', + }, + 'voting_power': power, + }, + 'election': { + 'node_id': f'node-{random.choice(range(100))}', + 'power': power, + 'public_key': { + 'value': base64.b16encode(base58.b58decode(kp.public_key)).decode('utf-8'), + 'type': 'ed25519-base16', + }, + }, + 'public_key': kp.public_key, + 'private_key': kp.private_key, + }) + return validators + + +def generate_election(b, cls, public_key, private_key, asset_data, voter_keys): + voters = cls.recipients(b) + election = cls.generate([public_key], + voters, + asset_data, + None).sign([private_key]) + + votes = [Vote.generate([election.to_inputs()[i]], + [([Election.to_public_key(election.id)], power)], + election.id) for i, (_, power) in enumerate(voters)] + for key, v in zip(voter_keys, votes): + v.sign([key]) + + return election, votes diff --git a/tests/validation/test_transaction_structure.py b/tests/validation/test_transaction_structure.py new file mode 100644 index 0000000..a15e823 --- /dev/null +++ b/tests/validation/test_transaction_structure.py @@ -0,0 +1,259 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +"""All tests of transaction structure. The concern here is that transaction +structural / schematic issues are caught when reading a transaction +(ie going from dict -> transaction). +""" +import json + +import pytest +try: + import hashlib as sha3 +except ImportError: + import sha3 +from unittest.mock import MagicMock + +from bigchaindb.common.exceptions import (AmountError, + SchemaValidationError, + ThresholdTooDeep) +from bigchaindb.models import Transaction + +################################################################################ +# Helper functions + + +def validate(tx): + if isinstance(tx, Transaction): + tx = tx.to_dict() + Transaction.from_dict(tx) + + +def validate_raises(tx, exc=SchemaValidationError): + with pytest.raises(exc): + validate(tx) + + +# We should test that validation works when we expect it to +def test_validation_passes(signed_create_tx): + Transaction.from_dict(signed_create_tx.to_dict()) + + +################################################################################ +# ID + + +def test_tx_serialization_hash_function(signed_create_tx): + tx = signed_create_tx.to_dict() + tx['id'] = None + payload = json.dumps(tx, skipkeys=False, sort_keys=True, + separators=(',', ':')) + assert sha3.sha3_256(payload.encode()).hexdigest() == signed_create_tx.id + + +def test_tx_serialization_with_incorrect_hash(signed_create_tx): + from bigchaindb.common.transaction import Transaction + from bigchaindb.common.exceptions import InvalidHash + tx = signed_create_tx.to_dict() + tx['id'] = 'a' * 64 + with pytest.raises(InvalidHash): + Transaction.validate_id(tx) + + +def test_tx_serialization_with_no_hash(signed_create_tx): + from bigchaindb.common.exceptions import InvalidHash + tx = signed_create_tx.to_dict() + del tx['id'] + with pytest.raises(InvalidHash): + Transaction.from_dict(tx) + + +################################################################################ +# Operation + +def test_validate_invalid_operation(b, create_tx, alice): + create_tx.operation = 'something invalid' + signed_tx = create_tx.sign([alice.private_key]) + validate_raises(signed_tx) + + +################################################################################ +# Metadata + +def test_validate_fails_metadata_empty_dict(b, create_tx, alice): + create_tx.metadata = {'a': 1} + signed_tx = create_tx.sign([alice.private_key]) + validate(signed_tx) + + create_tx._id = None + create_tx.fulfillment = None + create_tx.metadata = None + signed_tx = create_tx.sign([alice.private_key]) + validate(signed_tx) + + create_tx._id = None + create_tx.fulfillment = None + create_tx.metadata = {} + signed_tx = create_tx.sign([alice.private_key]) + validate_raises(signed_tx) + + +################################################################################ +# Asset + +def test_transfer_asset_schema(user_sk, signed_transfer_tx): + from bigchaindb.common.transaction import Transaction + tx = signed_transfer_tx.to_dict() + validate(tx) + tx['id'] = None + tx['asset']['data'] = {} + tx = Transaction.from_dict(tx).sign([user_sk]).to_dict() + validate_raises(tx) + tx['id'] = None + del tx['asset']['data'] + tx['asset']['id'] = 'b' * 63 + tx = Transaction.from_dict(tx).sign([user_sk]).to_dict() + validate_raises(tx) + + +def test_create_tx_no_asset_id(b, create_tx, alice): + create_tx.asset['id'] = 'b' * 64 + signed_tx = create_tx.sign([alice.private_key]) + validate_raises(signed_tx) + + +def test_create_tx_asset_type(b, create_tx, alice): + create_tx.asset['data'] = 'a' + signed_tx = create_tx.sign([alice.private_key]) + validate_raises(signed_tx) + + +def test_create_tx_no_asset_data(b, create_tx, alice): + tx_body = create_tx.to_dict() + del tx_body['asset']['data'] + tx_serialized = json.dumps( + tx_body, skipkeys=False, sort_keys=True, separators=(',', ':')) + tx_body['id'] = sha3.sha3_256(tx_serialized.encode()).hexdigest() + validate_raises(tx_body) + + +################################################################################ +# Inputs + +def test_no_inputs(b, create_tx, alice): + create_tx.inputs = [] + signed_tx = create_tx.sign([alice.private_key]) + validate_raises(signed_tx) + + +def test_create_single_input(b, create_tx, alice): + from bigchaindb.common.transaction import Transaction + tx = create_tx.to_dict() + tx['inputs'] += tx['inputs'] + tx = Transaction.from_dict(tx).sign([alice.private_key]).to_dict() + validate_raises(tx) + tx['id'] = None + tx['inputs'] = [] + tx = Transaction.from_dict(tx).sign([alice.private_key]).to_dict() + validate_raises(tx) + + +def test_create_tx_no_fulfills(b, create_tx, alice): + from bigchaindb.common.transaction import Transaction + tx = create_tx.to_dict() + tx['inputs'][0]['fulfills'] = {'transaction_id': 'a' * 64, + 'output_index': 0} + tx = Transaction.from_dict(tx).sign([alice.private_key]).to_dict() + validate_raises(tx) + + +def test_transfer_has_inputs(user_sk, signed_transfer_tx, alice): + signed_transfer_tx.inputs = [] + signed_transfer_tx._id = None + signed_transfer_tx.sign([user_sk]) + validate_raises(signed_transfer_tx) + + +################################################################################ +# Outputs + +def test_low_amounts(b, user_sk, create_tx, signed_transfer_tx, alice): + for sk, tx in [(alice.private_key, create_tx), (user_sk, signed_transfer_tx)]: + tx.outputs[0].amount = 0 + tx._id = None + tx.sign([sk]) + validate_raises(tx, AmountError) + tx.outputs[0].amount = -1 + tx._id = None + tx.sign([sk]) + validate_raises(tx) + + +def test_high_amounts(b, create_tx, alice): + # Should raise a SchemaValidationError - don't want to allow ridiculously + # large numbers to get converted to int + create_tx.outputs[0].amount = 10 ** 21 + create_tx.sign([alice.private_key]) + validate_raises(create_tx) + # Should raise AmountError + create_tx.outputs[0].amount = 9 * 10 ** 18 + 1 + create_tx._id = None + create_tx.sign([alice.private_key]) + validate_raises(create_tx, AmountError) + # Should pass + create_tx.outputs[0].amount -= 1 + create_tx._id = None + create_tx.sign([alice.private_key]) + validate(create_tx) + + +################################################################################ +# Conditions + +def test_handle_threshold_overflow(): + from bigchaindb.common import transaction + + cond = { + 'type': 'ed25519-sha-256', + 'public_key': 'a' * 43, + } + for i in range(1000): + cond = { + 'type': 'threshold-sha-256', + 'threshold': 1, + 'subconditions': [cond], + } + with pytest.raises(ThresholdTooDeep): + transaction._fulfillment_from_details(cond) + + +def test_unsupported_condition_type(): + from bigchaindb.common import transaction + from cryptoconditions.exceptions import UnsupportedTypeError + + with pytest.raises(UnsupportedTypeError): + transaction._fulfillment_from_details({'type': 'a'}) + + with pytest.raises(UnsupportedTypeError): + transaction._fulfillment_to_details(MagicMock(type_name='a')) + + +################################################################################ +# Version + +def test_validate_version(b, create_tx, alice): + create_tx.version = '2.0' + create_tx.sign([alice.private_key]) + validate(create_tx) + + create_tx.version = '0.10' + create_tx._id = None + create_tx.sign([alice.private_key]) + validate_raises(create_tx) + + create_tx.version = '110' + create_tx._id = None + create_tx.sign([alice.private_key]) + validate_raises(create_tx) diff --git a/tests/web/__init__.py b/tests/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/web/conftest.py b/tests/web/conftest.py new file mode 100644 index 0000000..2e0afcc --- /dev/null +++ b/tests/web/conftest.py @@ -0,0 +1,19 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + + +@pytest.fixture +def app(request): + from bigchaindb.web import server + from bigchaindb.lib import BigchainDB + + if request.config.getoption('--database-backend') == 'localmongodb': + app = server.create_app(debug=True, bigchaindb_factory=Planetmint) + else: + app = server.create_app(debug=True) + + return app diff --git a/tests/web/test_assets.py b/tests/web/test_assets.py new file mode 100644 index 0000000..b16749b --- /dev/null +++ b/tests/web/test_assets.py @@ -0,0 +1,72 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + +ASSETS_ENDPOINT = '/api/v1/assets/' + + +def test_get_assets_with_empty_text_search(client): + res = client.get(ASSETS_ENDPOINT + '?search=') + assert res.json == {'status': 400, + 'message': 'text_search cannot be empty'} + assert res.status_code == 400 + + +def test_get_assets_with_missing_text_search(client): + res = client.get(ASSETS_ENDPOINT) + assert res.status_code == 400 + + +@pytest.mark.bdb +def test_get_assets_tendermint(client, b, alice): + from bigchaindb.models import Transaction + + # test returns empty list when no assets are found + res = client.get(ASSETS_ENDPOINT + '?search=abc') + assert res.json == [] + assert res.status_code == 200 + + # create asset + asset = {'msg': 'abc'} + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset).sign([alice.private_key]) + + b.store_bulk_transactions([tx]) + + # test that asset is returned + res = client.get(ASSETS_ENDPOINT + '?search=abc') + assert res.status_code == 200 + assert len(res.json) == 1 + assert res.json[0] == { + 'data': {'msg': 'abc'}, + 'id': tx.id + } + + +@pytest.mark.bdb +def test_get_assets_limit_tendermint(client, b, alice): + from bigchaindb.models import Transaction + + # create two assets + asset1 = {'msg': 'abc 1'} + asset2 = {'msg': 'abc 2'} + tx1 = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset1).sign([alice.private_key]) + tx2 = Transaction.create([alice.public_key], [([alice.public_key], 1)], + asset=asset2).sign([alice.private_key]) + + b.store_bulk_transactions([tx1]) + b.store_bulk_transactions([tx2]) + + # test that both assets are returned without limit + res = client.get(ASSETS_ENDPOINT + '?search=abc') + assert res.status_code == 200 + assert len(res.json) == 2 + + # test that only one asset is returned when using limit=1 + res = client.get(ASSETS_ENDPOINT + '?search=abc&limit=1') + assert res.status_code == 200 + assert len(res.json) == 1 diff --git a/tests/web/test_block_tendermint.py b/tests/web/test_block_tendermint.py new file mode 100644 index 0000000..a11210d --- /dev/null +++ b/tests/web/test_block_tendermint.py @@ -0,0 +1,74 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + +from bigchaindb.models import Transaction +from bigchaindb.lib import Block + +BLOCKS_ENDPOINT = '/api/v1/blocks/' + + +@pytest.mark.bdb +@pytest.mark.usefixtures('inputs') +def test_get_block_endpoint(b, client, alice): + import copy + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset={'cycle': 'hero'}) + tx = tx.sign([alice.private_key]) + + # with store_bulk_transactions we use `insert_many` where PyMongo + # automatically adds an `_id` field to the tx, therefore we need the + # deepcopy, for more info see: + # https://api.mongodb.com/python/current/faq.html#writes-and-ids + tx_dict = copy.deepcopy(tx.to_dict()) + b.store_bulk_transactions([tx]) + + block = Block(app_hash='random_utxo', + height=31, + transactions=[tx.id]) + b.store_block(block._asdict()) + + res = client.get(BLOCKS_ENDPOINT + str(block.height)) + expected_response = {'height': block.height, 'transactions': [tx_dict]} + assert res.json == expected_response + assert res.status_code == 200 + + +@pytest.mark.bdb +@pytest.mark.usefixtures('inputs') +def test_get_block_returns_404_if_not_found(client): + res = client.get(BLOCKS_ENDPOINT + '123') + assert res.status_code == 404 + + res = client.get(BLOCKS_ENDPOINT + '123/') + assert res.status_code == 404 + + +@pytest.mark.bdb +def test_get_block_containing_transaction(b, client, alice): + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset={'cycle': 'hero'}) + tx = tx.sign([alice.private_key]) + b.store_bulk_transactions([tx]) + + block = Block(app_hash='random_utxo', + height=13, + transactions=[tx.id]) + b.store_block(block._asdict()) + + res = client.get('{}?transaction_id={}'.format(BLOCKS_ENDPOINT, tx.id)) + expected_response = [block.height] + assert res.json == expected_response + assert res.status_code == 200 + + +@pytest.mark.bdb +def test_get_blocks_by_txid_endpoint_returns_empty_list_not_found(client): + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=') + assert res.status_code == 200 + assert len(res.json) == 0 + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123') + assert res.status_code == 200 + assert len(res.json) == 0 diff --git a/tests/web/test_blocks.py b/tests/web/test_blocks.py new file mode 100644 index 0000000..54aa134 --- /dev/null +++ b/tests/web/test_blocks.py @@ -0,0 +1,55 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + +BLOCKS_ENDPOINT = '/api/v1/blocks/' + + +@pytest.mark.bdb +@pytest.mark.usefixtures('inputs') +def test_get_block_returns_404_if_not_found(client): + res = client.get(BLOCKS_ENDPOINT + '123') + assert res.status_code == 404 + + res = client.get(BLOCKS_ENDPOINT + '123/') + assert res.status_code == 404 + + +@pytest.mark.bdb +def test_get_blocks_by_txid_endpoint_returns_empty_list_not_found(client): + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=') + assert res.status_code == 200 + assert len(res.json) == 0 + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123') + assert res.status_code == 200 + assert len(res.json) == 0 + + +@pytest.mark.bdb +def test_get_blocks_by_txid_endpoint_returns_400_bad_query_params(client): + res = client.get(BLOCKS_ENDPOINT) + assert res.status_code == 400 + + res = client.get(BLOCKS_ENDPOINT + '?ts_id=123') + assert res.status_code == 400 + assert res.json == { + 'message': { + 'transaction_id': 'Missing required parameter in the JSON body or the post body or the query string' + } + } + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123&foo=123') + assert res.status_code == 400 + assert res.json == { + 'message': 'Unknown arguments: foo' + } + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123&status=123') + assert res.status_code == 400 + assert res.json == { + 'message': 'Unknown arguments: status' + } diff --git a/tests/web/test_content_type_middleware.py b/tests/web/test_content_type_middleware.py new file mode 100644 index 0000000..579a9e0 --- /dev/null +++ b/tests/web/test_content_type_middleware.py @@ -0,0 +1,45 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from unittest.mock import Mock + +OUTPUTS_ENDPOINT = '/api/v1/outputs/' + + +def test_middleware_does_nothing_when_no_content_type_is_provided(): + from bigchaindb.web.strip_content_type_middleware import StripContentTypeMiddleware + mock = Mock() + middleware = StripContentTypeMiddleware(mock) + middleware({'REQUEST_METHOD': 'GET'}, None) + + assert 'CONTENT_TYPE' not in mock.call_args[0][0] + + +def test_middleware_strips_content_type_from_gets(): + from bigchaindb.web.strip_content_type_middleware import StripContentTypeMiddleware + mock = Mock() + middleware = StripContentTypeMiddleware(mock) + middleware({'REQUEST_METHOD': 'GET', + 'CONTENT_TYPE': 'application/json'}, + None) + + assert 'CONTENT_TYPE' not in mock.call_args[0][0] + + +def test_middleware_does_notstrip_content_type_from_other_methods(): + from bigchaindb.web.strip_content_type_middleware import StripContentTypeMiddleware + mock = Mock() + middleware = StripContentTypeMiddleware(mock) + middleware({'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'application/json'}, + None) + + assert 'CONTENT_TYPE' in mock.call_args[0][0] + + +def test_get_outputs_endpoint_with_content_type(client, user_pk): + res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk), + headers=[('Content-Type', 'application/json')]) + assert res.status_code == 200 diff --git a/tests/web/test_info.py b/tests/web/test_info.py new file mode 100644 index 0000000..039ef27 --- /dev/null +++ b/tests/web/test_info.py @@ -0,0 +1,52 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +from unittest import mock + + +@mock.patch('bigchaindb.version.__short_version__', 'tst') +@mock.patch('bigchaindb.version.__version__', 'tsttst') +def test_api_root_endpoint(client, wsserver_base_url): + res = client.get('/') + docs_url = ['https://docs.bigchaindb.com/projects/server/en/vtsttst', + '/http-client-server-api.html'] + assert res.json == { + 'api': { + 'v1': { + 'docs': ''.join(docs_url), + 'transactions': '/api/v1/transactions/', + 'blocks': '/api/v1/blocks/', + 'assets': '/api/v1/assets/', + 'outputs': '/api/v1/outputs/', + 'streams': '{}/api/v1/streams/valid_transactions'.format( + wsserver_base_url), + 'metadata': '/api/v1/metadata/', + 'validators': '/api/v1/validators', + } + }, + 'docs': 'https://docs.bigchaindb.com/projects/server/en/vtsttst/', + 'version': 'tsttst', + 'software': 'Planetmint', + } + + +@mock.patch('bigchaindb.version.__short_version__', 'tst') +@mock.patch('bigchaindb.version.__version__', 'tsttst') +def test_api_v1_endpoint(client, wsserver_base_url): + docs_url = ['https://docs.bigchaindb.com/projects/server/en/vtsttst', + '/http-client-server-api.html'] + api_v1_info = { + 'docs': ''.join(docs_url), + 'transactions': '/transactions/', + 'blocks': '/blocks/', + 'assets': '/assets/', + 'outputs': '/outputs/', + 'streams': '{}/api/v1/streams/valid_transactions'.format( + wsserver_base_url), + 'metadata': '/metadata/', + 'validators': '/validators' + } + res = client.get('/api/v1') + assert res.json == api_v1_info diff --git a/tests/web/test_metadata.py b/tests/web/test_metadata.py new file mode 100644 index 0000000..328ba82 --- /dev/null +++ b/tests/web/test_metadata.py @@ -0,0 +1,75 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + +METADATA_ENDPOINT = '/api/v1/metadata/' + + +def test_get_metadata_with_empty_text_search(client): + res = client.get(METADATA_ENDPOINT + '?search=') + assert res.json == {'status': 400, + 'message': 'text_search cannot be empty'} + assert res.status_code == 400 + + +def test_get_metadata_with_missing_text_search(client): + res = client.get(METADATA_ENDPOINT) + assert res.status_code == 400 + + +@pytest.mark.bdb +def test_get_metadata_tendermint(client, b, alice): + from bigchaindb.models import Transaction + + # test returns empty list when no assets are found + res = client.get(METADATA_ENDPOINT + '?search=abc') + assert res.json == [] + assert res.status_code == 200 + + # create asset + asset = {'msg': 'abc'} + metadata = {'key': 'my_meta'} + tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], metadata=metadata, + asset=asset).sign([alice.private_key]) + + b.store_bulk_transactions([tx]) + + # test that metadata is returned + res = client.get(METADATA_ENDPOINT + '?search=my_meta') + assert res.status_code == 200 + assert len(res.json) == 1 + assert res.json[0] == { + 'metadata': {'key': 'my_meta'}, + 'id': tx.id + } + + +@pytest.mark.bdb +def test_get_metadata_limit_tendermint(client, b, alice): + from bigchaindb.models import Transaction + + # create two assets + asset1 = {'msg': 'abc 1'} + meta1 = {'key': 'meta 1'} + tx1 = Transaction.create([alice.public_key], [([alice.public_key], 1)], metadata=meta1, + asset=asset1).sign([alice.private_key]) + b.store_bulk_transactions([tx1]) + + asset2 = {'msg': 'abc 2'} + meta2 = {'key': 'meta 2'} + tx2 = Transaction.create([alice.public_key], [([alice.public_key], 1)], metadata=meta2, + asset=asset2).sign([alice.private_key]) + b.store_bulk_transactions([tx2]) + + # test that both assets are returned without limit + res = client.get(METADATA_ENDPOINT + '?search=meta') + assert res.status_code == 200 + assert len(res.json) == 2 + + # test that only one asset is returned when using limit=1 + res = client.get(METADATA_ENDPOINT + '?search=meta&limit=1') + assert res.status_code == 200 + assert len(res.json) == 1 diff --git a/tests/web/test_outputs.py b/tests/web/test_outputs.py new file mode 100644 index 0000000..6def190 --- /dev/null +++ b/tests/web/test_outputs.py @@ -0,0 +1,140 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest +from unittest.mock import MagicMock, patch + + +OUTPUTS_ENDPOINT = '/api/v1/outputs/' + + +@pytest.mark.bdb +@pytest.mark.userfixtures('inputs') +def test_get_outputs_endpoint(client, user_pk): + m = MagicMock() + m.txid = 'a' + m.output = 0 + with patch('bigchaindb.Planetmint.get_outputs_filtered') as gof: + gof.return_value = [m, m] + res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk)) + assert res.json == [ + {'transaction_id': 'a', 'output_index': 0}, + {'transaction_id': 'a', 'output_index': 0} + ] + assert res.status_code == 200 + gof.assert_called_once_with(user_pk, None) + + +def test_get_outputs_endpoint_unspent(client, user_pk): + m = MagicMock() + m.txid = 'a' + m.output = 0 + with patch('bigchaindb.Planetmint.get_outputs_filtered') as gof: + gof.return_value = [m] + params = '?spent=False&public_key={}'.format(user_pk) + res = client.get(OUTPUTS_ENDPOINT + params) + assert res.json == [{'transaction_id': 'a', 'output_index': 0}] + assert res.status_code == 200 + gof.assert_called_once_with(user_pk, False) + + +@pytest.mark.bdb +@pytest.mark.userfixtures('inputs') +def test_get_outputs_endpoint_spent(client, user_pk): + m = MagicMock() + m.txid = 'a' + m.output = 0 + with patch('bigchaindb.Planetmint.get_outputs_filtered') as gof: + gof.return_value = [m] + params = '?spent=true&public_key={}'.format(user_pk) + res = client.get(OUTPUTS_ENDPOINT + params) + assert res.json == [{'transaction_id': 'a', 'output_index': 0}] + assert res.status_code == 200 + gof.assert_called_once_with(user_pk, True) + + +@pytest.mark.bdb +@pytest.mark.userfixtures('inputs') +def test_get_outputs_endpoint_without_public_key(client): + res = client.get(OUTPUTS_ENDPOINT) + assert res.status_code == 400 + + +@pytest.mark.bdb +@pytest.mark.userfixtures('inputs') +def test_get_outputs_endpoint_with_invalid_public_key(client): + expected = {'message': {'public_key': 'Invalid base58 ed25519 key'}} + res = client.get(OUTPUTS_ENDPOINT + '?public_key=abc') + assert expected == res.json + assert res.status_code == 400 + + +@pytest.mark.bdb +@pytest.mark.userfixtures('inputs') +def test_get_outputs_endpoint_with_invalid_spent(client, user_pk): + expected = {'message': {'spent': 'Boolean value must be "true" or "false" (lowercase)'}} + params = '?spent=tru&public_key={}'.format(user_pk) + res = client.get(OUTPUTS_ENDPOINT + params) + assert expected == res.json + assert res.status_code == 400 + + +@pytest.mark.abci +def test_get_divisble_transactions_returns_500(b, client): + from bigchaindb.models import Transaction + from bigchaindb.common import crypto + import json + + TX_ENDPOINT = '/api/v1/transactions' + + def mine(tx_list): + b.store_bulk_transactions(tx_list) + + alice_priv, alice_pub = crypto.generate_key_pair() + bob_priv, bob_pub = crypto.generate_key_pair() + carly_priv, carly_pub = crypto.generate_key_pair() + + create_tx = Transaction.create([alice_pub], [([alice_pub], 4)]) + create_tx.sign([alice_priv]) + + res = client.post(TX_ENDPOINT, data=json.dumps(create_tx.to_dict())) + assert res.status_code == 202 + + mine([create_tx]) + + transfer_tx = Transaction.transfer(create_tx.to_inputs(), + [([alice_pub], 3), ([bob_pub], 1)], + asset_id=create_tx.id) + transfer_tx.sign([alice_priv]) + + res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx.to_dict())) + assert res.status_code == 202 + + mine([transfer_tx]) + + transfer_tx_carly = Transaction.transfer([transfer_tx.to_inputs()[1]], + [([carly_pub], 1)], + asset_id=create_tx.id) + transfer_tx_carly.sign([bob_priv]) + + res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx_carly.to_dict())) + assert res.status_code == 202 + + mine([transfer_tx_carly]) + + asset_id = create_tx.id + + url = TX_ENDPOINT + '?asset_id=' + asset_id + assert client.get(url).status_code == 200 + assert len(client.get(url).json) == 3 + + url = OUTPUTS_ENDPOINT + '?public_key=' + alice_pub + assert client.get(url).status_code == 200 + + url = OUTPUTS_ENDPOINT + '?public_key=' + bob_pub + assert client.get(url).status_code == 200 + + url = OUTPUTS_ENDPOINT + '?public_key=' + carly_pub + assert client.get(url).status_code == 200 diff --git a/tests/web/test_parameters.py b/tests/web/test_parameters.py new file mode 100644 index 0000000..ab2b5ba --- /dev/null +++ b/tests/web/test_parameters.py @@ -0,0 +1,80 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import pytest + + +def test_valid_txid(): + from bigchaindb.web.views.parameters import valid_txid + + valid = ['18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e4', + '18AC3E7343F016890C510E93F935261169D9E3F565436429830FAF0934F4F8E4'] + for h in valid: + assert valid_txid(h) == h.lower() + + non = ['18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e', + '18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e45', + '18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8eg', + '18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e ', + ''] + for h in non: + with pytest.raises(ValueError): + valid_txid(h) + + +def test_valid_bool(): + from bigchaindb.web.views.parameters import valid_bool + + assert valid_bool('true') is True + assert valid_bool('false') is False + assert valid_bool('tRUE') is True + assert valid_bool('fALSE') is False + + with pytest.raises(ValueError): + valid_bool('0') + with pytest.raises(ValueError): + valid_bool('1') + with pytest.raises(ValueError): + valid_bool('yes') + with pytest.raises(ValueError): + valid_bool('no') + + +def test_valid_ed25519(): + from bigchaindb.web.views.parameters import valid_ed25519 + + valid = ['123456789abcdefghijkmnopqrstuvwxyz1111111111', + '123456789ABCDEFGHJKLMNPQRSTUVWXYZ1111111111'] + for h in valid: + assert valid_ed25519(h) == h + + with pytest.raises(ValueError): + valid_ed25519('1234556789abcdefghijkmnopqrstuvwxyz1111111') + with pytest.raises(ValueError): + valid_ed25519('1234556789abcdefghijkmnopqrstuvwxyz1111111111') + with pytest.raises(ValueError): + valid_ed25519('123456789abcdefghijkmnopqrstuvwxyz111111111l') + with pytest.raises(ValueError): + valid_ed25519('123456789abcdefghijkmnopqrstuvwxyz111111111I') + with pytest.raises(ValueError): + valid_ed25519('1234556789abcdefghijkmnopqrstuvwxyz11111111O') + with pytest.raises(ValueError): + valid_ed25519('1234556789abcdefghijkmnopqrstuvwxyz111111110') + + +def test_valid_operation(): + from bigchaindb.web.views.parameters import valid_operation + + assert valid_operation('create') == 'CREATE' + assert valid_operation('transfer') == 'TRANSFER' + assert valid_operation('CREATe') == 'CREATE' + assert valid_operation('TRANSFEr') == 'TRANSFER' + + with pytest.raises(ValueError): + valid_operation('GENESIS') + with pytest.raises(ValueError): + valid_operation('blah') + with pytest.raises(ValueError): + valid_operation('') diff --git a/tests/web/test_server.py b/tests/web/test_server.py new file mode 100644 index 0000000..75fd975 --- /dev/null +++ b/tests/web/test_server.py @@ -0,0 +1,15 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +def test_settings(): + import bigchaindb + from bigchaindb.web import server + + s = server.create_server(bigchaindb.config['server']) + + # for whatever reason the value is wrapped in a list + # needs further investigation + assert s.cfg.bind[0] == bigchaindb.config['server']['bind'] diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py new file mode 100644 index 0000000..cf8ba24 --- /dev/null +++ b/tests/web/test_transactions.py @@ -0,0 +1,459 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import json +from unittest.mock import Mock, patch + +import base58 +import pytest +from cryptoconditions import Ed25519Sha256 +try: + from hashlib import sha3_256 +except ImportError: + from sha3 import sha3_256 + +from bigchaindb.common import crypto +from bigchaindb.common.transaction_mode_types import (BROADCAST_TX_COMMIT, + BROADCAST_TX_ASYNC, + BROADCAST_TX_SYNC) + +TX_ENDPOINT = '/api/v1/transactions/' + + +@pytest.mark.abci +def test_get_transaction_endpoint(client, posted_create_tx): + res = client.get(TX_ENDPOINT + posted_create_tx.id) + assert posted_create_tx.to_dict() == res.json + assert res.status_code == 200 + + +def test_get_transaction_returns_404_if_not_found(client): + res = client.get(TX_ENDPOINT + '123') + assert res.status_code == 404 + + res = client.get(TX_ENDPOINT + '123/') + assert res.status_code == 404 + + +@pytest.mark.abci +def test_post_create_transaction_endpoint(b, client): + from bigchaindb.models import Transaction + user_priv, user_pub = crypto.generate_key_pair() + + tx = Transaction.create([user_pub], [([user_pub], 1)]) + tx = tx.sign([user_priv]) + + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + + assert res.status_code == 202 + + assert res.json['inputs'][0]['owners_before'][0] == user_pub + assert res.json['outputs'][0]['public_keys'][0] == user_pub + + +@pytest.mark.abci +@pytest.mark.parametrize('nested', [False, True]) +@pytest.mark.parametrize('language,expected_status_code', [ + ('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202), + ('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202), + ('norwegian', 202), ('portuguese', 202), ('romanian', 202), ('none', 202), + ('russian', 202), ('spanish', 202), ('swedish', 202), ('turkish', 202), + ('da', 202), ('nl', 202), ('en', 202), ('fi', 202), ('fr', 202), + ('de', 202), ('hu', 202), ('it', 202), ('nb', 202), ('pt', 202), + ('ro', 202), ('ru', 202), ('es', 202), ('sv', 202), ('tr', 202), + ('any', 400) +]) +@pytest.mark.language +def test_post_create_transaction_with_language(b, client, nested, language, + expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection + + if isinstance(b.connection, LocalMongoDBConnection): + user_priv, user_pub = crypto.generate_key_pair() + lang_obj = {'language': language} + + if nested: + asset = {'root': lang_obj} + else: + asset = lang_obj + + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset=asset) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + 'Invalid transaction (ValidationError): MongoDB does not support ' + 'text search for the language "{}". If you do not understand this ' + 'error message then please rename key/field "language" to something ' + 'else like "lang".').format(language) + assert res.json['message'] == expected_error_message + + +@pytest.mark.abci +@pytest.mark.parametrize('field', ['asset', 'metadata']) +@pytest.mark.parametrize('value,err_key,expected_status_code', [ + ({'bad.key': 'v'}, 'bad.key', 400), + ({'$bad.key': 'v'}, '$bad.key', 400), + ({'$badkey': 'v'}, '$badkey', 400), + ({'bad\x00key': 'v'}, 'bad\x00key', 400), + ({'good_key': {'bad.key': 'v'}}, 'bad.key', 400), + ({'good_key': 'v'}, 'good_key', 202) +]) +def test_post_create_transaction_with_invalid_key(b, client, field, value, + err_key, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection + user_priv, user_pub = crypto.generate_key_pair() + + if isinstance(b.connection, LocalMongoDBConnection): + if field == 'asset': + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset=value) + elif field == 'metadata': + tx = Transaction.create([user_pub], [([user_pub], 1)], + metadata=value) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + + assert res.status_code == expected_status_code + + if res.status_code == 400: + expected_error_message = ( + 'Invalid transaction (ValidationError): Invalid key name "{}" ' + 'in {} object. The key name cannot contain characters ' + '".", "$" or null characters').format(err_key, field) + assert res.json['message'] == expected_error_message + + +@pytest.mark.abci +@patch('bigchaindb.web.views.base.logger') +def test_post_create_transaction_with_invalid_id(mock_logger, b, client): + from bigchaindb.common.exceptions import InvalidHash + from bigchaindb.models import Transaction + user_priv, user_pub = crypto.generate_key_pair() + + tx = Transaction.create([user_pub], [([user_pub], 1)]) + tx = tx.sign([user_priv]).to_dict() + tx['id'] = 'abcd' * 16 + + res = client.post(TX_ENDPOINT, data=json.dumps(tx)) + expected_status_code = 400 + expected_error_message = ( + "Invalid transaction ({}): The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid." + ).format(InvalidHash.__name__, tx['id']) + assert res.status_code == expected_status_code + assert res.json['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + { + 'message': expected_error_message, 'status': expected_status_code, + 'method': 'POST', 'path': TX_ENDPOINT + } in mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[0].args['status'] == expected_status_code + # assert caplog.records[0].args['message'] == expected_error_message + + +@pytest.mark.abci +@patch('bigchaindb.web.views.base.logger') +def test_post_create_transaction_with_invalid_signature(mock_logger, + b, + client): + from bigchaindb.common.exceptions import InvalidSignature + from bigchaindb.models import Transaction + user_priv, user_pub = crypto.generate_key_pair() + + tx = Transaction.create([user_pub], [([user_pub], 1)]).to_dict() + tx['inputs'][0]['fulfillment'] = 64 * '0' + tx['id'] = sha3_256( + json.dumps( + tx, + sort_keys=True, + separators=(',', ':'), + ensure_ascii=False, + ).encode(), + ).hexdigest() + + res = client.post(TX_ENDPOINT, data=json.dumps(tx)) + expected_status_code = 400 + expected_error_message = ( + 'Invalid transaction ({}): Fulfillment URI ' + 'couldn\'t been parsed' + ).format(InvalidSignature.__name__) + assert res.status_code == expected_status_code + assert res.json['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + { + 'message': expected_error_message, 'status': expected_status_code, + 'method': 'POST', 'path': TX_ENDPOINT + } in mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[0].args['status'] == expected_status_code + # assert caplog.records[0].args['message'] == expected_error_message + + +@pytest.mark.abci +def test_post_create_transaction_with_invalid_structure(client): + res = client.post(TX_ENDPOINT, data='{}') + assert res.status_code == 400 + + +@pytest.mark.abci +@patch('bigchaindb.web.views.base.logger') +def test_post_create_transaction_with_invalid_schema(mock_logger, client): + from bigchaindb.models import Transaction + user_priv, user_pub = crypto.generate_key_pair() + tx = Transaction.create([user_pub], [([user_pub], 1)]).to_dict() + del tx['version'] + ed25519 = Ed25519Sha256(public_key=base58.b58decode(user_pub)) + message = json.dumps( + tx, + sort_keys=True, + separators=(',', ':'), + ensure_ascii=False, + ).encode() + ed25519.sign(message, base58.b58decode(user_priv)) + tx['inputs'][0]['fulfillment'] = ed25519.serialize_uri() + tx['id'] = sha3_256( + json.dumps( + tx, + sort_keys=True, + separators=(',', ':'), + ensure_ascii=False, + ).encode(), + ).hexdigest() + res = client.post(TX_ENDPOINT, data=json.dumps(tx)) + expected_status_code = 400 + expected_error_message = ( + "Invalid transaction schema: 'version' is a required property") + assert res.status_code == expected_status_code + assert res.json['message'] == expected_error_message + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + { + 'message': expected_error_message, 'status': expected_status_code, + 'method': 'POST', 'path': TX_ENDPOINT + } in mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[0].args['status'] == expected_status_code + # assert caplog.records[0].args['message'] == expected_error_message + + +@pytest.mark.abci +@pytest.mark.parametrize('exc,msg', ( + ('AmountError', 'Do the math again!'), + ('DoubleSpend', 'Nope! It is gone now!'), + ('InvalidHash', 'Do not smoke that!'), + ('InvalidSignature', 'Falsche Unterschrift!'), + ('ValidationError', 'Create and transfer!'), + ('InputDoesNotExist', 'Hallucinations?'), + ('TransactionOwnerError', 'Not yours!'), + ('ValidationError', '?'), +)) +@patch('bigchaindb.web.views.base.logger') +def test_post_invalid_transaction(mock_logger, client, exc, msg, monkeypatch,): + from bigchaindb.common import exceptions + exc_cls = getattr(exceptions, exc) + + def mock_validation(self_, tx): + raise exc_cls(msg) + + TransactionMock = Mock(validate=mock_validation) + + monkeypatch.setattr( + 'bigchaindb.models.Transaction.from_dict', lambda tx: TransactionMock) + res = client.post(TX_ENDPOINT, data=json.dumps({})) + expected_status_code = 400 + expected_error_message = 'Invalid transaction ({}): {}'.format(exc, msg) + assert res.status_code == expected_status_code + assert (res.json['message'] == + 'Invalid transaction ({}): {}'.format(exc, msg)) + assert mock_logger.error.called + assert ( + 'HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s' in + mock_logger.error.call_args[0] + ) + assert ( + { + 'message': expected_error_message, 'status': expected_status_code, + 'method': 'POST', 'path': TX_ENDPOINT + } in mock_logger.error.call_args[0] + ) + # TODO put back caplog based asserts once possible + # assert caplog.records[2].args['status'] == expected_status_code + # assert caplog.records[2].args['message'] == expected_error_message + + +@pytest.mark.abci +def test_post_transfer_transaction_endpoint(client, user_pk, user_sk, posted_create_tx): + from bigchaindb.models import Transaction + + transfer_tx = Transaction.transfer(posted_create_tx.to_inputs(), + [([user_pk], 1)], + asset_id=posted_create_tx.id) + transfer_tx = transfer_tx.sign([user_sk]) + + res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx.to_dict())) + + assert res.status_code == 202 + + assert res.json['inputs'][0]['owners_before'][0] == user_pk + assert res.json['outputs'][0]['public_keys'][0] == user_pk + + +@pytest.mark.abci +def test_post_invalid_transfer_transaction_returns_400(client, user_pk, posted_create_tx): + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import InvalidSignature + + transfer_tx = Transaction.transfer(posted_create_tx.to_inputs(), + [([user_pk], 1)], + asset_id=posted_create_tx.id) + transfer_tx._hash() + + res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx.to_dict())) + expected_status_code = 400 + expected_error_message = 'Invalid transaction ({}): {}'.format( + InvalidSignature.__name__, 'Transaction signature is invalid.') + assert res.status_code == expected_status_code + assert res.json['message'] == expected_error_message + + +@pytest.mark.abci +def test_post_wrong_asset_division_transfer_returns_400(b, client, user_pk): + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import AmountError + + priv_key, pub_key = crypto.generate_key_pair() + + create_tx = Transaction.create([pub_key], + [([pub_key], 10)], + asset={'test': 'asset'}).sign([priv_key]) + res = client.post(TX_ENDPOINT + '?mode=commit', data=json.dumps(create_tx.to_dict())) + assert res.status_code == 202 + + transfer_tx = Transaction.transfer(create_tx.to_inputs(), + [([pub_key], 20)], # 20 > 10 + asset_id=create_tx.id).sign([priv_key]) + res = client.post(TX_ENDPOINT + '?mode=commit', data=json.dumps(transfer_tx.to_dict())) + expected_error_message = \ + f'Invalid transaction ({AmountError.__name__}): ' + \ + 'The amount used in the inputs `10` needs to be same as the amount used in the outputs `20`' + + assert res.status_code == 400 + assert res.json['message'] == expected_error_message + + +def test_transactions_get_list_good(client): + from functools import partial + + def get_txs_patched(conn, **args): + """Patch `get_transactions_filtered` so that rather than return an array + of transactions it returns an array of shims with a to_dict() method + that reports one of the arguments passed to `get_transactions_filtered`. + """ + return [type('', (), {'to_dict': partial(lambda a: a, arg)}) + for arg in sorted(args.items())] + + asset_id = '1' * 64 + + with patch('bigchaindb.Planetmint.get_transactions_filtered', get_txs_patched): + url = TX_ENDPOINT + '?asset_id=' + asset_id + assert client.get(url).json == [ + ['asset_id', asset_id], + ['last_tx', None], + ['operation', None] + ] + url = TX_ENDPOINT + '?asset_id=' + asset_id + '&operation=CREATE' + assert client.get(url).json == [ + ['asset_id', asset_id], + ['last_tx', None], + ['operation', 'CREATE'] + ] + url = TX_ENDPOINT + '?asset_id=' + asset_id + '&last_tx=true' + assert client.get(url).json == [ + ['asset_id', asset_id], + ['last_tx', True], + ['operation', None] + ] + + +def test_transactions_get_list_bad(client): + def should_not_be_called(): + assert False + with patch('bigchaindb.Planetmint.get_transactions_filtered', + lambda *_, **__: should_not_be_called()): + # Test asset id validated + url = TX_ENDPOINT + '?asset_id=' + '1' * 63 + assert client.get(url).status_code == 400 + # Test operation validated + url = TX_ENDPOINT + '?asset_id=' + '1' * 64 + '&operation=CEATE' + assert client.get(url).status_code == 400 + # Test asset ID required + url = TX_ENDPOINT + '?operation=CREATE' + assert client.get(url).status_code == 400 + + +@patch('requests.post') +@pytest.mark.parametrize('mode', [ + ('', BROADCAST_TX_ASYNC), + ('?mode=async', BROADCAST_TX_ASYNC), + ('?mode=sync', BROADCAST_TX_SYNC), + ('?mode=commit', BROADCAST_TX_COMMIT), +]) +def test_post_transaction_valid_modes(mock_post, client, mode): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + + def _mock_post(*args, **kwargs): + return Mock(json=Mock(return_value={'result': {'code': 0}})) + + mock_post.side_effect = _mock_post + + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]) + mode_endpoint = TX_ENDPOINT + mode[0] + client.post(mode_endpoint, data=json.dumps(tx.to_dict())) + args, kwargs = mock_post.call_args + assert mode[1] == kwargs['json']['method'] + + +@pytest.mark.abci +def test_post_transaction_invalid_mode(client): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]) + mode_endpoint = TX_ENDPOINT + '?mode=nope' + response = client.post(mode_endpoint, data=json.dumps(tx.to_dict())) + assert '400 BAD REQUEST' in response.status + assert 'Mode must be "async", "sync" or "commit"' ==\ + json.loads(response.data.decode('utf8'))['message']['mode'] diff --git a/tests/web/test_validators.py b/tests/web/test_validators.py new file mode 100644 index 0000000..304273d --- /dev/null +++ b/tests/web/test_validators.py @@ -0,0 +1,23 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +VALIDATORS_ENDPOINT = '/api/v1/validators/' + + +def test_get_validators_endpoint(b, client): + validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2', + 'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A', + 'type': 'ed25519'}, + 'voting_power': 10}] + b.store_validator_set(23, validator_set) + + res = client.get(VALIDATORS_ENDPOINT) + assert is_validator(res.json[0]) + assert res.status_code == 200 + + +# Helper +def is_validator(v): + return ('pub_key' in v) and ('voting_power' in v) diff --git a/tests/web/test_websocket_server.py b/tests/web/test_websocket_server.py new file mode 100644 index 0000000..8c4f526 --- /dev/null +++ b/tests/web/test_websocket_server.py @@ -0,0 +1,211 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import asyncio +import json +import queue +import threading +from unittest.mock import patch + +import pytest + + +class MockWebSocket: + def __init__(self): + self.received = [] + + def send_str(self, s): + self.received.append(s) + + +def test_eventify_block_works_with_any_transaction(): + from bigchaindb.web.websocket_server import eventify_block + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.lib import Transaction + + alice = generate_key_pair() + + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)])\ + .sign([alice.private_key]) + tx_transfer = Transaction.transfer(tx.to_inputs(), + [([alice.public_key], 1)], + asset_id=tx.id)\ + .sign([alice.private_key]) + + block = {'height': 1, + 'transactions': [tx, tx_transfer]} + + expected_events = [{ + 'height': 1, + 'asset_id': tx.id, + 'transaction_id': tx.id + }, { + 'height': 1, + 'asset_id': tx_transfer.asset['id'], + 'transaction_id': tx_transfer.id + }] + + for event, expected in zip(eventify_block(block), expected_events): + assert event == expected + + +async def test_bridge_sync_async_queue(loop): + from bigchaindb.web.websocket_server import _multiprocessing_to_asyncio + + sync_queue = queue.Queue() + async_queue = asyncio.Queue(loop=loop) + + bridge = threading.Thread(target=_multiprocessing_to_asyncio, + args=(sync_queue, async_queue, loop), + daemon=True) + bridge.start() + + sync_queue.put('fahren') + sync_queue.put('auf') + sync_queue.put('der') + sync_queue.put('Autobahn') + + result = await async_queue.get() + assert result == 'fahren' + + result = await async_queue.get() + assert result == 'auf' + + result = await async_queue.get() + assert result == 'der' + + result = await async_queue.get() + assert result == 'Autobahn' + + assert async_queue.qsize() == 0 + + +@patch('threading.Thread') +@patch('aiohttp.web.run_app') +@patch('bigchaindb.web.websocket_server.init_app') +@patch('asyncio.get_event_loop', return_value='event-loop') +@patch('asyncio.Queue', return_value='event-queue') +def test_start_creates_an_event_loop(queue_mock, get_event_loop_mock, + init_app_mock, run_app_mock, + thread_mock): + from bigchaindb import config + from bigchaindb.web.websocket_server import start, _multiprocessing_to_asyncio + + start(None) + thread_mock.assert_called_once_with( + target=_multiprocessing_to_asyncio, + args=(None, queue_mock.return_value, get_event_loop_mock.return_value), + daemon=True, + ) + thread_mock.return_value.start.assert_called_once_with() + init_app_mock.assert_called_with('event-queue', loop='event-loop') + run_app_mock.assert_called_once_with( + init_app_mock.return_value, + host=config['wsserver']['host'], + port=config['wsserver']['port'], + ) + + +async def test_websocket_string_event(test_client, loop): + from bigchaindb.web.websocket_server import init_app, POISON_PILL, EVENTS_ENDPOINT + + event_source = asyncio.Queue(loop=loop) + app = init_app(event_source, loop=loop) + client = await test_client(app) + ws = await client.ws_connect(EVENTS_ENDPOINT) + + await event_source.put('hack') + await event_source.put('the') + await event_source.put('planet!') + + result = await ws.receive() + assert result.data == 'hack' + + result = await ws.receive() + assert result.data == 'the' + + result = await ws.receive() + assert result.data == 'planet!' + + await event_source.put(POISON_PILL) + + +async def test_websocket_block_event(b, test_client, loop): + from bigchaindb import events + from bigchaindb.web.websocket_server import init_app, POISON_PILL, EVENTS_ENDPOINT + from bigchaindb.models import Transaction + from bigchaindb.common import crypto + + user_priv, user_pub = crypto.generate_key_pair() + tx = Transaction.create([user_pub], [([user_pub], 1)]) + tx = tx.sign([user_priv]) + + event_source = asyncio.Queue(loop=loop) + app = init_app(event_source, loop=loop) + client = await test_client(app) + ws = await client.ws_connect(EVENTS_ENDPOINT) + block = {'height': 1, 'transactions': [tx]} + block_event = events.Event(events.EventTypes.BLOCK_VALID, block) + + await event_source.put(block_event) + + for tx in block['transactions']: + result = await ws.receive() + json_result = json.loads(result.data) + assert json_result['transaction_id'] == tx.id + # Since the transactions are all CREATEs, asset id == transaction id + assert json_result['asset_id'] == tx.id + assert json_result['height'] == block['height'] + + await event_source.put(POISON_PILL) + + +@pytest.mark.skip('Processes are not stopping properly, and the whole test suite would hang') +def test_integration_from_webapi_to_websocket(monkeypatch, client, loop): + # XXX: I think that the `pytest-aiohttp` plugin is sparkling too much + # magic in the `asyncio` module: running this test without monkey-patching + # `asycio.get_event_loop` (and without the `loop` fixture) raises a: + # RuntimeError: There is no current event loop in thread 'MainThread'. + # + # That's pretty weird because this test doesn't use the pytest-aiohttp + # plugin explicitely. + monkeypatch.setattr('asyncio.get_event_loop', lambda: loop) + + import json + import random + import aiohttp + + from bigchaindb.common import crypto + # TODO processes does not exist anymore, when reactivating this test it + # will fail because of this + from bigchaindb import processes + from bigchaindb.models import Transaction + + # Start Planetmint + processes.start() + + loop = asyncio.get_event_loop() + + import time + time.sleep(1) + + ws_url = client.get('http://localhost:9984/api/v1/').json['_links']['streams_v1'] + + # Connect to the WebSocket endpoint + session = aiohttp.ClientSession() + ws = loop.run_until_complete(session.ws_connect(ws_url)) + + # Create a keypair and generate a new asset + user_priv, user_pub = crypto.generate_key_pair() + asset = {'random': random.random()} + tx = Transaction.create([user_pub], [([user_pub], 1)], asset=asset) + tx = tx.sign([user_priv]) + # Post the transaction to the Planetmint Web API + client.post('/api/v1/transactions/', data=json.dumps(tx.to_dict())) + + result = loop.run_until_complete(ws.receive()) + json_result = json.loads(result.data) + assert json_result['transaction_id'] == tx.id diff --git a/tmdata/config.toml b/tmdata/config.toml new file mode 100644 index 0000000..e03a079 --- /dev/null +++ b/tmdata/config.toml @@ -0,0 +1,18 @@ +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +proxy_app = "tcp://bigchaindb:26658" +moniker = "anonymous" +fast_sync = true +db_backend = "leveldb" +log_level = "state:debug,*:error" + +[consensus] +create_empty_blocks = false + +[rpc] +laddr = "tcp://0.0.0.0:26657" + +[p2p] +laddr = "tcp://0.0.0.0:26656" +seeds = "" diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..c983d4d --- /dev/null +++ b/tox.ini @@ -0,0 +1,36 @@ +[tox] +skipsdist = true +envlist = py{36,37,38}, flake8, docsroot + +[base] +basepython = python3.8 +deps = pip>=9.0.1 + +[testenv] +usedevelop = True +setenv = + PYTHONPATH={toxinidir}:{toxinidir}/bigchaindb +deps = {[base]deps} +install_command = pip install {opts} {packages} +extras = test +commands = pytest -v -n auto --cov=bigchaindb --basetemp={envtmpdir} + +[testenv:flake8] +basepython = {[base]basepython} +deps = + {[base]deps} + flake8 +skip_install = True +extras = None +commands = flake8 bigchaindb tests + +[testenv:docsroot] +basepython = {[base]basepython} +changedir = docs/root/source +deps = + {[base]deps} + typing-extensions + -r{toxinidir}/docs/root/requirements.txt +extras = None +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +