Capitalisation mistake (#2734)

Changed "BigchaindB" to "BigchainDB"

.ci/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+
+set -e -x
+
+if [[ ${BIGCHAINDB_CI_ABCI} == 'enable' ]]; then
+    sleep 3600
+else
+    bigchaindb -l DEBUG start
+fi

.ci/travis-after-success.sh

@@ -1,7 +1,12 @@
 #!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 
 set -e -x
 
-if [[ -z ${TOXENV} ]]; then
-    codecov
+if [[ -z ${TOXENV} ]] && [[ ${BIGCHAINDB_CI_ABCI} != 'enable' ]] && [[ ${BIGCHAINDB_ACCEPTANCE_TEST} != 'enable' ]]; then
+    codecov -v -f htmlcov/coverage.xml
 fi

.ci/travis-before-install.sh

@@ -1,6 +1,16 @@
 #!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
 
-apt-get update -qq
-wget https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz -O /tmp/v1.1.0.tar.gz
-tar -xvf /tmp/v1.1.0.tar.gz
-cp -r $PWD/rapidjson-1.1.0/include/rapidjson /usr/include/
+
+if [[ -z ${TOXENV} ]]; then
+  sudo apt-get update
+  sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+
+  sudo rm /usr/local/bin/docker-compose
+  curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  chmod +x docker-compose
+  sudo mv docker-compose /usr/local/bin
+fi

.ci/travis-before-script.sh

@@ -1,34 +1,18 @@
 #!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 
 set -e -x
 
-if [[ "${TOXENV}" == *-rdb ]]; then
-    rethinkdb --daemon
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \
-        -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then
-    # Connect to MongoDB on port 27017 via a normal, unsecure connection if
-    # BIGCHAINDB_DATABASE_SSL is unset.
-    # It is unset in this case in .travis.yml.
-    wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1404-3.4.4.tgz -O /tmp/mongodb.tgz
-    tar -xvf /tmp/mongodb.tgz
-    mkdir /tmp/mongodb-data
-    ${PWD}/mongodb-linux-x86_64-ubuntu1404-3.4.4/bin/mongod \
-        --dbpath=/tmp/mongodb-data --replSet=bigchain-rs &> /dev/null &
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \
-        "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then
-    # Connect to MongoDB on port 27017 via TLS/SSL connection if
-    # BIGCHAINDB_DATABASE_SSL is set.
-    # It is set to 'true' here in .travis.yml. Dummy certificates for testing
-    # are stored under bigchaindb/tests/backend/mongodb-ssl/certs/ directory.
-    wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1404-3.4.4.tgz -O /tmp/mongodb-ssl.tgz
-    tar -xvf /tmp/mongodb-ssl.tgz
-    mkdir /tmp/mongodb-ssl-data
-    ${PWD}/mongodb-linux-x86_64-ubuntu1404-3.4.4/bin/mongod \
-        --dbpath=/tmp/mongodb-ssl-data \
-        --replSet=bigchain-rs \
-        --sslAllowInvalidHostnames \
-        --sslMode=requireSSL \
-        --sslCAFile=$TRAVIS_BUILD_DIR/tests/backend/mongodb-ssl/certs/ca.crt \
-        --sslCRLFile=$TRAVIS_BUILD_DIR/tests/backend/mongodb-ssl/certs/crl.pem \
-        --sslPEMKeyFile=$TRAVIS_BUILD_DIR/tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem &> /dev/null &
+if [[ -z ${TOXENV} ]]; then
+
+  if [[ ${BIGCHAINDB_CI_ABCI} == 'enable' ]]; then
+      docker-compose up -d bigchaindb
+  else
+      docker-compose up -d bdb
+  fi
+
 fi

.ci/travis-install.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 
 set -e -x
 
@@ -6,7 +11,11 @@ pip install --upgrade pip
 
 if [[ -n ${TOXENV} ]]; then
     pip install --upgrade tox
+elif [[ ${BIGCHAINDB_CI_ABCI} == 'enable' ]]; then
+    docker-compose build --no-cache --build-arg abci_status=enable bigchaindb
+elif [[ $BIGCHAINDB_INTEGRATION_TEST == 'enable' ]]; then
+    docker-compose build bigchaindb python-driver
 else
-    pip install .[test]
+    docker-compose build --no-cache bigchaindb
     pip install --upgrade codecov
 fi

.ci/travis_script.sh

@@ -1,17 +1,18 @@
 #!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 
 set -e -x
 
 if [[ -n ${TOXENV} ]]; then
   tox -e ${TOXENV}
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \
-    -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then
-    # Run the full suite of tests for MongoDB over an unsecure connection
-  pytest -sv --database-backend=mongodb --cov=bigchaindb
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \
-    "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then
-    # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'.
-  pytest -sv --database-backend=mongodb-ssl --cov=bigchaindb -m bdb_ssl
+elif [[ ${BIGCHAINDB_CI_ABCI} == 'enable' ]]; then
+  docker-compose exec bigchaindb pytest -v -m abci
+elif [[ ${BIGCHAINDB_ACCEPTANCE_TEST} == 'enable' ]]; then
+    ./run-acceptance-test.sh
 else
-  pytest -sv -n auto --cov=bigchaindb
+  docker-compose exec bigchaindb pytest -v --cov=bigchaindb --cov-report xml:htmlcov/coverage.xml
 fi

.gitattributes

@@ -1,9 +0,0 @@
-deploy-cluster-aws export-ignore
-docs export-ignore
-ntools export-ignore
-tests export-ignore
-.gitattributes export-ignore
-.gitignore export-ignore
-.travis.yml export-ignore
-*.md export-ignore
-codecov.yml export-ignore

.github/CONTRIBUTING.md

@@ -0,0 +1,14 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
+# How to Contribute to the BigchainDB Project
+
+There are many ways you can contribute to the BigchainDB project, some very easy and others more involved.
+
+All of that is documented elsewhere: go to the "[Contributing to BigchainDB" docs on ReadTheDocs](https://docs.bigchaindb.com/projects/contributing/en/latest/index.html).
+
+Note: GitHub automatically links to this file (`.github/CONTRIBUTING.md`) when a contributor creates a new issue or pull request, so you shouldn't delete it. Just use it to point people to full and proper help elsewhere.

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,20 @@
+# Do you want to:
+
+- make a bug report? Then read below about what should go in a bug report.
+- make a feature request or proposal? Then read [the page about how to make a feature request or proposal](https://docs.bigchaindb.com/projects/contributing/en/latest/ways-can-contribute/make-a-feature-request-or-proposal.html).
+- ask a question about BigchainDB? Then [go to Gitter](https://gitter.im/bigchaindb/bigchaindb) (our chat room) and ask it there.
+- share your neat idea or realization? Then [go to Gitter](https://gitter.im/bigchaindb/bigchaindb) (our chat room) and share it there.
+
+# What Should Go in a Bug Report
+
+- What computer are you on (hardware)?
+- What operating system are you using, including version. e.g. Ubuntu 14.04? Fedora 23?
+- What version of BigchainDB software were you using? Is that the latest version?
+- What, exactly, did you do to get to the point where you got stuck? Describe all the steps so we can get there too. Show screenshots or copy-and-paste text to GitHub.
+- Show what actually happened.
+- Say what you tried to do to resolve the problem.
+- Provide details to convince us that it matters to you. Is it for a school project, a job, a contract with a deadline, a child who needs it for Christmas?
+
+We will do our best but please understand that we don't have time to help everyone, especially people who don't care to help us help them. "It doesn't work." is not going to get any reaction from us. We need _details_.
+
+Tip: Use Github code block formatting to make code render pretty in GitHub. To do that, put three backticks followed by a string to set the type of code (e.g. `Python`), then the code, and then end with three backticks. There's more information about [inserting code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks/) in the GitHub help pages.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs or terminal output**
+If applicable, add add textual content to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - Distribution: [e.g. Ubuntu 18.04]
+ - Bigchaindb version:
+ - Tendermint version:
+ - Mongodb version:
+- Python full version: [e.g. Python 3.6.6]
+
+**Additional context**
+Add any other context about the problem here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,18 @@
+Make sure the title of this pull request has the form:
+
+**Problem: A short statement of the problem.**
+
+## Solution
+
+A short statement about how this PR solves the **Problem**.
+
+## Issues Resolved
+
+What issues does this PR resolve, if any? Please include lines like the following (i.e. "Resolves #NNNN), so that when this PR gets merged, GitHub will automatically close those issues.
+
+Resolves #NNNN
+Resolves #MMMM
+
+## BEPs Implemented
+
+What [BEPs](https://github.com/bigchaindb/beps) does this pull request implement, if any?

.gitignore

@@ -6,6 +6,21 @@ __pycache__/
 # C extensions
 *.so
 
+# Swap -- copypasta from https://github.com/github/gitignore/blob/master/Global/Vim.gitignore
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-v][a-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+
 # Distribution / packaging
 .Python
 env/
@@ -40,6 +55,7 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
+.pytest_cache/
 nosetests.xml
 coverage.xml
 *.cover
@@ -65,22 +81,19 @@ target/
 # pyenv
 .python-version
 
-# Some files created when deploying a cluster on AWS
-deploy-cluster-aws/conf/rethinkdb.conf
-deploy-cluster-aws/confiles/
-deploy-cluster-aws/client_confile
-deploy-cluster-aws/hostlist.py
-deploy-cluster-aws/ssh_key.py
-
-# Ansible-specific files
-ntools/one-m/ansible/hosts
-ntools/one-m/ansible/ansible.cfg
-
 # Just in time documentation
-docs/server/source/schema
 docs/server/source/http-samples
 
 # Terraform state files
 # See https://stackoverflow.com/a/41482391
 terraform.tfstate
 terraform.tfstate.backup
+
+# tendermint data
+tmdata/data
+network/*/data
+
+# Docs that are fetched at build time
+docs/contributing/source/cross-project-policies/*.md
+
+.DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+repos:
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: v1.1.1
+    hooks:
+    -   id: trailing-whitespace
+        args: ['--no-markdown-linebreak-ext']
+    -   id: check-merge-conflict
+    -   id: debug-statements
+    -   id: check-added-large-files
+    -   id: flake8
+
+-   repo: git://github.com/chewse/pre-commit-mirrors-pydocstyle
+    sha: v2.1.1
+    hooks:
+    -   id: pydocstyle
+        # list of error codes to check, see: http://www.pydocstyle.org/en/latest/error_codes.html
+        args: ['--select=D204,D201,D209,D210,D212,D300,D403']
+
+# negate the exclude to only apply the hooks to 'bigchaindb' and 'tests' folder
+exclude: '^(?!bigchaindb/)(?!tests/)(?!acceptance/)'

.readthedocs.yml

@@ -0,0 +1,11 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+build:
+    image: latest
+
+python:
+    version: 3.6
+    pip_install: true

.travis.yml

@@ -1,49 +1,70 @@
+# Copyright © 2020, 2021 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+sudo: required
+
+dist: focal
+
+services:
+  - docker
+
 language: python
 cache: pip
 
 python:
-  - 3.5
   - 3.6
-  
+  - 3.7
+  - 3.8
+
 env:
-  - TOXENV=flake8
-  - TOXENV=docsroot
-  - TOXENV=docsserver
+  global:
+    - DOCKER_COMPOSE_VERSION=1.29.2
+  matrix:
+    - TOXENV=flake8
+    - TOXENV=docsroot
 
 matrix:
   fast_finish: true
-  exclude:
-  - python: 3.5
-    env: TOXENV=flake8 
-  - python: 3.5 
-    env: TOXENV=docsroot
-  - python: 3.5 
-    env: TOXENV=docsserver
   include:
-  - python: 3.5
-    addons: 
-      rethinkdb: '2.3.5'
-    env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb
-  - python: 3.5
+  - python: 3.6
     env:
-    - BIGCHAINDB_DATABASE_BACKEND=mongodb
+    - BIGCHAINDB_DATABASE_BACKEND=localmongodb
     - BIGCHAINDB_DATABASE_SSL=
-  - python: 3.6
-    addons: 
-      rethinkdb: '2.3.5'
-    env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb
   - python: 3.6
     env:
-    - BIGCHAINDB_DATABASE_BACKEND=mongodb
+    - BIGCHAINDB_DATABASE_BACKEND=localmongodb
     - BIGCHAINDB_DATABASE_SSL=
-  - python: 3.5
-    env:
-    - BIGCHAINDB_DATABASE_BACKEND=mongodb
-    - BIGCHAINDB_DATABASE_SSL=true
+    - BIGCHAINDB_CI_ABCI=enable
   - python: 3.6
     env:
-    - BIGCHAINDB_DATABASE_BACKEND=mongodb
-    - BIGCHAINDB_DATABASE_SSL=true
+    - BIGCHAINDB_ACCEPTANCE_TEST=enable
+  - python: 3.7
+    env:
+    - BIGCHAINDB_DATABASE_BACKEND=localmongodb
+    - BIGCHAINDB_DATABASE_SSL=
+  - python: 3.7
+    env:
+    - BIGCHAINDB_DATABASE_BACKEND=localmongodb
+    - BIGCHAINDB_DATABASE_SSL=
+    - BIGCHAINDB_CI_ABCI=enable
+  - python: 3.7
+    env:
+    - BIGCHAINDB_ACCEPTANCE_TEST=enable
+  - python: 3.8
+    env:
+    - BIGCHAINDB_DATABASE_BACKEND=localmongodb
+    - BIGCHAINDB_DATABASE_SSL=
+  - python: 3.8
+    env:
+    - BIGCHAINDB_DATABASE_BACKEND=localmongodb
+    - BIGCHAINDB_DATABASE_SSL=
+    - BIGCHAINDB_CI_ABCI=enable
+  - python: 3.8
+    env:
+    - BIGCHAINDB_ACCEPTANCE_TEST=enable
+
 
 before_install: sudo .ci/travis-before-install.sh
 

CHANGELOG.md

@@ -1,9 +1,18 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Change Log (Release Notes)
 
 All _notable_ changes to this project will be documented in this file (`CHANGELOG.md`).
-This project adheres to [the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) (or at least we try).
-Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/).
-Note that each version (or "release") is the name of a [Git _tag_](https://git-scm.com/book/en/v2/Git-Basics-Tagging) of a particular commit, so the associated date and time are the date and time of that commit (as reported by GitHub), _not_ the "Uploaded on" date listed on PyPI (which may differ).
+
+This project adheres to [the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme) (or at least we try). The BigchainDB public API _was_ defined in this file but that definition was moved and can now be found in [BEP-7](https://github.com/bigchaindb/BEPs/tree/master/7).
+
+Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/). Note that each version (or "release") is the name of a [Git _tag_](https://git-scm.com/book/en/v2/Git-Basics-Tagging) of a particular commit, so the associated date and time are the date and time of that commit (as reported by GitHub), _not_ the "Uploaded on" date listed on PyPI (which may differ).
+
 For reference, the possible headings are:
 
 * **Added** for new features.
@@ -13,8 +22,439 @@ For reference, the possible headings are:
 * **Fixed** for any bug fixes.
 * **Security** to invite users to upgrade in case of vulnerabilities.
 * **External Contributors** to list contributors outside of BigchainDB GmbH.
+* **Known Issues**
 * **Notes**
 
+## [2.2.2] - 2020-08-12
+
+### Security
+
+Several dependencies updated including Flask that had vulnerability.
+
+### Fixed
+
+* Updated priv_validator key format in stack script (#2707)
+
+### External Contributors
+
+* @aostrun - [#2708](https://github.com/bigchaindb/bigchaindb/pull/2708)
+
+## [2.2.1] - 2020-04-14
+
+### Fixed
+
+Gevent library API update is incompatible with  bigchaindb-abci 1.0.1 version.
+Updated bigchaindb-abci.
+
+## [2.2.0] - 2020-02-20
+
+### Added
+
+Support for multiple ABCI versions.
+
+## [2.1.0] - 2019-11-06
+
+### Added
+
+Option for last transaction retrieval added.
+
+## [2.0] - 2019-09-26
+
+### Changed
+
+Migrated from Tendermint 0.22.8 to 0.31.5.
+
+## [2.0 Beta 9] - 2018-11-27
+
+### Changed
+
+Removed support for TLSv1 and TLSv1.1 in all NGINX config files. Kept support for TLSv1.2 and added support for TLSv1.3. [Pull Request #2601](https://github.com/bigchaindb/bigchaindb/pull/2601)
+
+### Fixed
+
+Fixed two issues with schema validation. Pull requests [#2606](https://github.com/bigchaindb/bigchaindb/pull/2606) & [#2607](https://github.com/bigchaindb/bigchaindb/pull/2607)
+
+### External Contributors
+
+[@gamjapark](https://github.com/gamjapark) and team translated all the [BigchainDB root docs](https://docs.bigchaindb.com/en/latest/korean/index.html) into Korean. [Pull Request #2603](https://github.com/bigchaindb/bigchaindb/pull/2603)
+
+## [2.0 Beta 8] - 2018-11-03
+
+### Changed
+
+* Revised the [Simple Deployment Template](http://docs.bigchaindb.com/projects/server/en/latest/simple-deployment-template/index.html) in the docs. Added NGINX to the mix. Pull Requests [#2578](https://github.com/bigchaindb/bigchaindb/pull/2578) and [#2579](https://github.com/bigchaindb/bigchaindb/pull/2579)
+* Revised `nginx/nginx.conf` to enable CORS. [Pull Request #2580](https://github.com/bigchaindb/bigchaindb/pull/2580)
+
+### Fixed
+
+* Fixed a typo in the Kubernetes ConfigMap template. [Pull Request #2583](https://github.com/bigchaindb/bigchaindb/pull/2583)
+
+### External Contributors
+
+[@gamjapark](https://github.com/gamjapark) translated the main `README.md` file into Korean. [Pull Request #2592](https://github.com/bigchaindb/bigchaindb/pull/2592)
+
+## [2.0 Beta 7] - 2018-09-28
+
+Tag name: v2.0.0b7
+
+### Added
+
+Completed the implementation of chain-migration elections (BEP-42). Pull requests [#2553](https://github.com/bigchaindb/bigchaindb/pull/2553), [#2556](https://github.com/bigchaindb/bigchaindb/pull/2556), [#2558](https://github.com/bigchaindb/bigchaindb/pull/2558), [#2563](https://github.com/bigchaindb/bigchaindb/pull/2563) and [#2566](https://github.com/bigchaindb/bigchaindb/pull/2566)
+
+### Changed
+
+* Code that used the Python driver's (deprecated) transactions.send() method now uses its transactions.send_commit() method instead. [Pull request #2547](https://github.com/bigchaindb/bigchaindb/pull/2547)
+* Code that implied pluggable "consensus" now implies pluggable transaction "validation" (a more accurate word). [Pull request #2561](https://github.com/bigchaindb/bigchaindb/pull/2561)
+
+### Removed
+
+Benchmark logs. [Pull request #2565](https://github.com/bigchaindb/bigchaindb/pull/2565)
+
+### Fixed
+
+A bug caused by an incorrect MongoDB query. [Pull request #2567](https://github.com/bigchaindb/bigchaindb/pull/2567)
+
+### Notes
+
+There's now better documentation about logs, log rotation, and the `server.bind` config setting. Pull requests [#2546](https://github.com/bigchaindb/bigchaindb/pull/2546) and [#2575](https://github.com/bigchaindb/bigchaindb/pull/2575)
+
+## [2.0 Beta 6] - 2018-09-17
+
+Tag name: v2.0.0b6
+
+### Added
+
+* [New documentation about privacy and handling private data](https://docs.bigchaindb.com/en/latest/private-data.html). [Pull request #2437](https://github.com/bigchaindb/bigchaindb/pull/2437)
+* New documentation about log rotation. Also rotate Tendermint logs if started using Monit. [Pull request #2528](https://github.com/bigchaindb/bigchaindb/pull/2528)
+* Began implementing one of the migration strategies outlined in [BEP-42](https://github.com/bigchaindb/BEPs/tree/master/42). That involved creating a more general-purpose election process and commands. Pull requests [#2488](https://github.com/bigchaindb/bigchaindb/pull/2488), [#2495](https://github.com/bigchaindb/bigchaindb/pull/2495), [#2498](https://github.com/bigchaindb/bigchaindb/pull/2498), [#2515](https://github.com/bigchaindb/bigchaindb/pull/2515), [#2535](https://github.com/bigchaindb/bigchaindb/pull/2535)
+* Used memoization to avoid doing some validation checks multiple times. [Pull request #2490](https://github.com/bigchaindb/bigchaindb/pull/2490)
+* Created an all-in-one Docker image containing BigchainDB Server, Tendermint and MongoDB. It was created for a particular user and is not recommended for production use unless you really know what you're doing. [Pull request #2424](https://github.com/bigchaindb/bigchaindb/pull/2424)
+
+### Changed
+
+* The supported versions of Tendermint are now hard-wired into BigchainDB Server: it checks to see what version the connected Tendermint has, and if it's not compatible, BigchainDB Server exits with an error message. [Pull request #2541](https://github.com/bigchaindb/bigchaindb/pull/2541)
+* The docs no longer say to install the highest version of Tendermint: they say to install a specific version. [Pull request #2524](https://github.com/bigchaindb/bigchaindb/pull/2524)
+* The setup docs include more recommended settings for `config.toml`. [Pull request #2516](https://github.com/bigchaindb/bigchaindb/pull/2516)
+* The process to add, remove or update the voting power of a validator at run time (using the `bigchaindb upsert-validator` subcommands) was completely changed and is now fully working. See [issue #2372](https://github.com/bigchaindb/bigchaindb/issues/2372) and all the pull requests it references. Pull requests [#2439](https://github.com/bigchaindb/bigchaindb/pull/2439) and [#2440](https://github.com/bigchaindb/bigchaindb/pull/2440)
+* The license on the documentation was changed from CC-BY-SA-4 to CC-BY-4. [Pull request #2427](https://github.com/bigchaindb/bigchaindb/pull/2427)
+* Re-activated and/or updated some unit tests that had been deacivated during the migration to Tendermint. Pull requests [#2390](https://github.com/bigchaindb/bigchaindb/pull/2390), [#2415](https://github.com/bigchaindb/bigchaindb/pull/2415), [#2452](https://github.com/bigchaindb/bigchaindb/pull/24), [#2456](https://github.com/bigchaindb/bigchaindb/pull/2456)
+* Updated RapidJSON to a newer, faster version. [Pull request #2470](https://github.com/bigchaindb/bigchaindb/pull/2470)
+* The Java driver is now officially supported. [Pull request #2478](https://github.com/bigchaindb/bigchaindb/pull/2478)
+* The MongoDB indexes on transaction id and block height were changed to be [unique indexes](https://docs.mongodb.com/manual/core/index-unique/). [Pull request #2492](https://github.com/bigchaindb/bigchaindb/pull/2492)
+* Updated the required `cryptoconditions` package to a newer one. [Pull request #2494](https://github.com/bigchaindb/bigchaindb/pull/2494)
+
+### Removed
+
+* Removed some old code and tests. Pull requests
+  [#2374](https://github.com/bigchaindb/bigchaindb/pull/2374),
+  [#2452](https://github.com/bigchaindb/bigchaindb/pull/2452),
+  [#2474](https://github.com/bigchaindb/bigchaindb/pull/2474),
+  [#2476](https://github.com/bigchaindb/bigchaindb/pull/2476),
+  [#2491](https://github.com/bigchaindb/bigchaindb/pull/2491)
+
+### Fixed
+
+* Fixed the Events API so that it only sends valid transactions to subscribers. Also changed how it works internally, so now it is more reliable. [Pull request #2529](https://github.com/bigchaindb/bigchaindb/pull/2529)
+* Fixed a bug where MongoDB database initialization would abort if a collection already existed. [Pull request #2520](https://github.com/bigchaindb/bigchaindb/pull/2520)
+* Fixed a unit test that was failing randomly. [Pull request #2423](https://github.com/bigchaindb/bigchaindb/pull/2423)
+* Fixed the validator curl port. [Pull request #2447](https://github.com/bigchaindb/bigchaindb/pull/2447)
+* Fixed an error in the docs about the HTTP POST /transactions endpoint. [Pull request #2481](https://github.com/bigchaindb/bigchaindb/pull/2481)
+* Fixed a unit test that could loop forever. [Pull requqest #2486](https://github.com/bigchaindb/bigchaindb/pull/2486)
+* Fixed a bug when validating a CREATE + TRANSFER. [Pull request #2487](https://github.com/bigchaindb/bigchaindb/pull/2487)
+* Fixed the HTTP response when posting a transaction in commit mode. [Pull request #2510](https://github.com/bigchaindb/bigchaindb/pull/2510)
+* Fixed a crash that happened when attempting to restart BigchainDB at Tendermint block height 1. [Pull request#2519](https://github.com/bigchaindb/bigchaindb/pull/2519)
+
+### External Contributors
+
+@danacr - [Pull request #2447](https://github.com/bigchaindb/bigchaindb/pull/2447)
+
+### Notes
+
+The docs section titled "Production Deployment Template" was renamed to "Kubernetes Deployment Template" and we no longer consider it the go-to deployment template. The "Simple Deployment Template" is simpler, easier to understand, and less expensive (unless you are with an organization that already has a big Kubernetes cluster).
+
+## [2.0 Beta 5] - 2018-08-01
+
+Tag name: v2.0.0b5
+
+### Changed
+
+* Supported version of Tendermint `0.22.3` -> `0.22.8`. [Pull request #2429](https://github.com/bigchaindb/bigchaindb/pull/2429).
+
+### Fixed
+
+* Stateful validation raises a DoubleSpend exception if there is any other transaction that spends the same output(s) even if it has the same transaction ID. [Pull request #2422](https://github.com/bigchaindb/bigchaindb/pull/2422).
+
+## [2.0 Beta 4] - 2018-07-30
+
+Tag name: v2.0.0b4
+
+### Added
+
+- Added scripts for creating a configuration to manage processes with Monit. [Pull request #2410](https://github.com/bigchaindb/bigchaindb/pull/2410).
+
+### Fixed
+
+- Redundant asset and metadata queries were removed. [Pull request #2409](https://github.com/bigchaindb/bigchaindb/pull/2409).
+- Signal handling was fixed for BigchainDB processes. [Pull request #2395](https://github.com/bigchaindb/bigchaindb/pull/2395).
+- Some of the abruptly closed sockets that used to stay in memory are being cleaned up now. [Pull request 2408](https://github.com/bigchaindb/bigchaindb/pull/2408).
+- Fixed the bug when WebSockets powering Events API became unresponsive. [Pull request #2413](https://github.com/bigchaindb/bigchaindb/pull/2413).
+
+### Notes:
+
+* The instructions on how to write a BEP were simplified. [Pull request #2347](https://github.com/bigchaindb/bigchaindb/pull/2347).
+* A section about troubleshooting was added to the network setup guide. [Pull request #2398](https://github.com/bigchaindb/bigchaindb/pull/2398).
+* Some of the core code was given a better package structure. [Pull request #2401](https://github.com/bigchaindb/bigchaindb/pull/2401).
+* Some of the previously disabled unit tests were re-enabled and updated. Pull requests [#2404](https://github.com/bigchaindb/bigchaindb/pull/2404) and [#2402](https://github.com/bigchaindb/bigchaindb/pull/2402).
+* Some building blocks for dynamically adding new validators were introduced. [Pull request #2392](https://github.com/bigchaindb/bigchaindb/pull/2392).
+
+## [2.0 Beta 3] - 2018-07-18
+
+Tag name: v2.0.0b3
+
+### Fixed
+
+Fixed a bug in transaction validation. For some more-complex situations, it would say that a valid transaction was invalid. This bug was actually fixed before; it was [issue #1271](https://github.com/bigchaindb/bigchaindb/issues/1271). The unit test for it was turned off while we integrated Tendermint. Then the query implementation code got changed, reintroducing the bug, but the unit test was off so the bug wasn't caught. When we turned the test back on, shortly after releasing Beta 2, it failed, unveiling the bug. [Pull request #2389](https://github.com/bigchaindb/bigchaindb/pull/2389)
+
+## [2.0 Beta 2] - 2018-07-16
+
+Tag name: v2.0.0b2
+
+### Added
+
+* Added new configuration settings `tendermint.host` and `tendermint.port`. [Pull request #2342](https://github.com/bigchaindb/bigchaindb/pull/2342)
+* Added tests to ensure that BigchainDB gracefully handles "nasty" strings in keys and values. [Pull request #2334](https://github.com/bigchaindb/bigchaindb/pull/2334)
+* Added a new logging handler to capture benchmark stats to a separate file. [Pull request #2349](https://github.com/bigchaindb/bigchaindb/pull/2349)
+
+### Changed
+
+* Changed the names of BigchainDB processes (Python processes) to include 'bigchaindb', so they are easier to spot and find. [Pull request #2354](https://github.com/bigchaindb/bigchaindb/pull/2354)
+* Updated all code to support the latest version of Tendermint. Note that the BigchainDB ABCI server now listens to port 26657 instead of 46657. Pull requests [#2375](https://github.com/bigchaindb/bigchaindb/pull/2375) and [#2380](https://github.com/bigchaindb/bigchaindb/pull/2380)
+
+### Removed
+
+Removed all support and code for the old backlog_reassign_delay setting. [Pull request #2332](https://github.com/bigchaindb/bigchaindb/pull/2332)
+
+### Fixed
+
+* Fixed a bug that sometimes arose when using Docker Compose. (Tendermint would freeze.) [Pull request #2341](https://github.com/bigchaindb/bigchaindb/pull/2341)
+* Fixed a bug in the code that creates a MongoDB index for the "id" in the transactions collection. It works now, and performance is improved. [Pull request #2378](https://github.com/bigchaindb/bigchaindb/pull/2378)
+* The logging server would keep runnning in some tear-down scenarios. It doesn't do that any more. [Pull request #2304](https://github.com/bigchaindb/bigchaindb/pull/2304)
+
+### External Contributors
+
+@hrntknr - [Pull request #2331](https://github.com/bigchaindb/bigchaindb/pull/2331)
+
+### Known Issues
+
+The `bigchaindb upsert-validator` subcommand is not working yet, but a solution ([BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21)) has been finalized and will be implemented before we release the final BigchainDB 2.0.
+
+### Notes
+
+* A lot of old/dead code was deleted. Pull requests
+[#2319](https://github.com/bigchaindb/bigchaindb/pull/2319),
+[#2338](https://github.com/bigchaindb/bigchaindb/pull/2338),
+[#2357](https://github.com/bigchaindb/bigchaindb/pull/2357),
+[#2365](https://github.com/bigchaindb/bigchaindb/pull/2365),
+[#2366](https://github.com/bigchaindb/bigchaindb/pull/2366),
+[#2368](https://github.com/bigchaindb/bigchaindb/pull/2368) and
+[#2374](https://github.com/bigchaindb/bigchaindb/pull/2374)
+* Improved the documentation page "How to setup a BigchainDB Network". [Pull Request #2312](https://github.com/bigchaindb/bigchaindb/pull/2312)
+
+## [2.0 Beta 1] - 2018-06-01
+
+Tag name: v2.0.0b1
+
+### Fixed
+
+* Fixed a bug that arose with some code that treated transactions-waiting-for-block-inclusion as if they were already stored in MongoDB (i.e. already in a block). [Pull request #2318](https://github.com/bigchaindb/bigchaindb/pull/2318)
+* If a user asked for a block and it happened to be an empty block, BigchainDB returned 404 Not Found, as if the block did not exist. Now it returns a 200 OK with a block containing no transactions, i.e. an empty block. [Pull request #2321](https://github.com/bigchaindb/bigchaindb/pull/2321)
+
+### Known Issues
+
+* An issue was found with the `bigchaindb upsert-validator` command. A solution was proposed in [BEP-19](https://github.com/bigchaindb/BEPs/pull/45) and is being implemented in [pull request #2314](https://github.com/bigchaindb/bigchaindb/pull/2314)
+* If you run BigchainDB locally using `make start` (i.e. using Docker Compose) and then you put the node under heavy write load, Tendermint can become unresponsive and running `make stop` can hang.
+* There seems to be one or more issues with Tendermint when it is put under heavy load (i.e. even when BigchainDB isn't involved). See Tendermint issues [#1394](https://github.com/tendermint/tendermint/issues/1394), [#1642](https://github.com/tendermint/tendermint/issues/1642) and [#1661](https://github.com/tendermint/tendermint/issues/1661)
+
+### Notes
+
+* There's a [new docs page](https://docs.bigchaindb.com/projects/server/en/v2.0.0b1/simple-network-setup.html) about how to set up a network where each node uses a single virtual machine.
+* We checked, and BigchainDB 2.0 Beta 1 works with MongoDB 3.6 (and 3.4).
+* Support for RethinkDB is completely gone.
+
+## [2.0 Alpha 6] - 2018-05-17
+
+Tag name: v2.0.0a6
+
+### Changed
+
+Upgraded PyMongo to version 3.6 (which is compatible with MongoDB 3.6, 3.4 [and more](https://docs.mongodb.com/ecosystem/drivers/driver-compatibility-reference/#python-driver-compatibility)). [Pull request #2298](https://github.com/bigchaindb/bigchaindb/pull/2298)
+
+### Fixed
+
+When deploying a node using our Docker Compose file, it didn't expose port 46656, which is used by Tendermint for inter-node communications, so the node couldn't communicate with other nodes. We fixed that in [pull request #2299](https://github.com/bigchaindb/bigchaindb/pull/2299)
+
+### Notes
+
+We ran all our tests using MongoDB 3.6 and they all passed, so it seems safe to use BigchainDB with MongoDB 3.6 from now on.
+
+## [2.0 Alpha 5] - 2018-05-11
+
+Tag name: v2.0.0a5
+
+### Changed
+
+To resolve [issue #2279](https://github.com/bigchaindb/bigchaindb/issues/2279), we made some changes to the `bigchaindb-abci` package (which is our fork of `py-abci`) and told BigchainDB to use the new version (`bigchaindb-abci==0.4.5`). [Pull request #2281](https://github.com/bigchaindb/bigchaindb/pull/2281).
+
+## [2.0 Alpha 4] - 2018-05-09
+
+Tag name: v2.0.0a4
+
+### Changed
+
+The Kubernetes liveness probe for the BigchainDB StatefulSet was improved to check the Tendermint /status endpoint in addition to the Tendermint /abci_info endpoint. [Pull request #2275](https://github.com/bigchaindb/bigchaindb/pull/2275)
+
+### Fixed
+
+[Pull request #2270](https://github.com/bigchaindb/bigchaindb/pull/2270) resolved [issue #2269](https://github.com/bigchaindb/bigchaindb/issues/2269).
+
+### Notes
+
+There's a new [page in the docs about storing files in BigchainDB](https://docs.bigchaindb.com/en/latest/store-files.html). [Pull request #2259](https://github.com/bigchaindb/bigchaindb/pull/2259)
+
+## [2.0 Alpha 3] - 2018-05-03
+
+Tag name: v2.0.0a3
+
+### Changed
+
+* Upgraded BigchainDB Server code to use the latest version of Tendermint: version 0.19.2. Pull requests [#2249](https://github.com/bigchaindb/bigchaindb/pull/2249), [#2252](https://github.com/bigchaindb/bigchaindb/pull/2252) and [#2253](https://github.com/bigchaindb/bigchaindb/pull/2253)
+* Made some fixes to `py-abci` (an external Python package) and used our fixed version with BigchainDB. Those fixes resolved several known issues, including [issue #2182](https://github.com/bigchaindb/bigchaindb/issues/2182) and issues with large transactions in general. Note: At the time of writing, our fixes to `py-abci` hadn't been merged into the main `py-abci` repository or its latest release on PyPI; we were using our own special `bigchaindb-abci` package (which included our fixes). Pull requests [#2250](https://github.com/bigchaindb/bigchaindb/pull/2250) and [#2261](https://github.com/bigchaindb/bigchaindb/pull/2261)
+* If BigchainDB Server crashes and then comes back, Tendermint Core doesn't try to reconnect to it. That's just how Tendermint Core works. We revised our Kubernetes-based production deployment template to resolve that issue: BigchainDB Server and Tendermint Core are now in the same Kubernetes StatefulSet; if the connection between them ever goes down, then Kubernetes restarts the whole StatefulSet. [Pull request #2242](https://github.com/bigchaindb/bigchaindb/pull/2242)
+
+### Fixed
+
+Re-enabled multi-threading. [Pull request #2258](https://github.com/bigchaindb/bigchaindb/pull/2258)
+
+### Known Issues
+
+Tendermint changed how it responds to a request to store data (via the [Tendermint Broadcast API](https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api)) between version 0.12 and 0.19.2. We started modifying the code of BigchainDB Server to account for those changes in responses (in [pull request #2239](https://github.com/bigchaindb/bigchaindb/pull/2239)), but we found that there's a difference between what the Tendermint documentation _says_ about those responses and how Tendermint actually responds. We need to determine Tendermint's intent before we can finalize that pull request.
+
+### Notes
+
+We were focused on getting the public BigchainDB Testnet stable during the development of BigchainDB 2.0 Alpha 3, and we think we largely succeeded. Because of that focus, we delayed the deployment of an internal test network until later. It would have had the same instabilities as the public BigchainDB Testnet anyway. In the future, we'll always test a new version of BigchainDB on our internal test network before deploying it on the public BigchainDB Testnet. (That wasn't possible this time around, because there was no old/stable version of BigchainDB 2.n to run on the public BigchainDB Testnet while we tested BigchainDB 2.[n+1] internally.)
+
+## [2.0 Alpha 2] - 2018-04-18
+
+Tag name: v2.0.0a2
+
+### Added
+
+An implementation of [BEP-8 (BigchainDB Enhancement Proposal #8)](https://github.com/bigchaindb/BEPs/tree/master/8), which makes sure a node can recover from a system fault (e.g. a crash) into a consistent state, i.e. a state where the data in the node's local MongoDB database is consistent with the data stored in the blockchain. Pull requests [#2135](https://github.com/bigchaindb/bigchaindb/pull/2135) and [#2207](https://github.com/bigchaindb/bigchaindb/pull/2207)
+
+### Changed
+
+When someone uses the HTTP API to send a new transaction to a BigchainDB network using the [POST /api/v1/transactions?mode={mode}](https://docs.bigchaindb.com/projects/server/en/master/http-client-server-api.html#post--api-v1-transactions?mode=mode) endpoint, they now get back a more informative HTTP response, so they can better-understand what happened. This is only when mode is `commit` or `sync`, because `async` _means_ that the response is immediate, without waiting to see what happened to the transaction. [Pull request #2198](https://github.com/bigchaindb/bigchaindb/pull/2198)
+
+### Known Issues
+
+* If BigchainDB Server crashes and then is restarted, Tendermint Core won't try to reconnect to BigchainDB Server and so all operations requiring that connection won't work. We only understood this recently. We'll write a blog post explaining what we intend to do about it.
+* The known issues in 2.0 Alpha (listed below) are still there.
+
+## [2.0 Alpha] - 2018-04-03
+
+Tag name: v2.0.0a1
+
+There were _many_ changes between 1.3 and 2.0 Alpha, too many to list here. We wrote a series of blog posts to summarize most changes, especially those that affect end users and application developers:
+
+* [BigchainDB 2.0 is Byzantine Fault Tolerant](https://blog.bigchaindb.com/bigchaindb-2-0-is-byzantine-fault-tolerant-5ffdac96bc44)
+* [Some HTTP API Changes in the Next Release](https://blog.bigchaindb.com/some-http-api-changes-in-the-next-release-49612a537b0c)
+* [Three Transaction Model Changes in the Next Release](https://blog.bigchaindb.com/three-transaction-model-changes-in-the-next-release-dadbac50094a)
+* [Changes to the Server Command Line Interface in BigchainDB 2.0](https://blog.bigchaindb.com/changes-to-the-server-command-line-interface-in-bigchaindb-2-0-e1d6576e7155)
+* _A forthcoming post about changes in BigchainDB Server configuration settings_
+
+### External Contributors
+
+* @r7vme contributed [pull request #1984](https://github.com/bigchaindb/bigchaindb/pull/1984) which fixed a bug in our Kubernetes-based production deployment template.
+
+### Known Issues
+
+We intend to resolve these issues before releasing the final BigchainDB 2.0:
+
+* There's a known Heisenbug that (sometimes) arises and we found that making the BigchainDB webserver single-threaded prevents that bug from causing problems. We intend to resolve that bug, but in the meantime our temporary workaround is to change the _default_ webserver configuration settings to single-threaded mode, i.e. `BIGCHAINDB_SERVER_WORKERS=1` and `BIGCHAINDB_SERVER_THREADS=1`.
+* Issues sometimes happen when a large transaction is sent to a BigchainDB network.
+
+## [1.3] - 2017-11-21
+Tag name: v1.3.0
+
+### Added
+* Metadata full-text search. [Pull request #1812](https://github.com/bigchaindb/bigchaindb/pull/1812)
+
+### Notes
+* Improved documentation about blocks and votes. [Pull request #1855](https://github.com/bigchaindb/bigchaindb/pull/1855)
+
+
+## [1.2] - 2017-11-13
+Tag name: v1.2.0
+
+### Added
+* New and improved installation setup docs and code. Pull requests [#1775](https://github.com/bigchaindb/bigchaindb/pull/1775) and [#1785](https://github.com/bigchaindb/bigchaindb/pull/1785)
+* New BigchainDB configuration setting to set the port number of the log server: `log.port`. [Pull request #1796](https://github.com/bigchaindb/bigchaindb/pull/1796)
+* New secondary index on `id` in the bigchain table. That will make some queries execute faster. [Pull request #1803](https://github.com/bigchaindb/bigchaindb/pull/1803)
+* When using MongoDB, there are some restrictions on allowed names for keys (JSON keys). Those restrictions were always there but now BigchainDB checks key names explicitly, rather than leaving that to MongoDB. Pull requests [#1807](https://github.com/bigchaindb/bigchaindb/pull/1807) and [#1811](https://github.com/bigchaindb/bigchaindb/pull/1811)
+* When using MongoDB, there are some restrictions on the allowed values of "language" (if that key is used in the values of `metadata` or `asset.data`). Those restrictions were always there but now BigchainDB checks the values explicitly, rather than leaving that to MongoDB. Pull requests [#1806](https://github.com/bigchaindb/bigchaindb/pull/1806) and [#1811](https://github.com/bigchaindb/bigchaindb/pull/1811)
+* There's a new page in the root docs about permissions in BigchainDB. [Pull request #1788](https://github.com/bigchaindb/bigchaindb/pull/1788)
+* There's a new option in the `bigchaindb start` command: `bigchaindb start --no-init` will avoid doing `bigchaindb init` if it wasn't done already. [Pull request #1814](https://github.com/bigchaindb/bigchaindb/pull/1814)
+
+### Fixed
+* Fixed a bug where setting the log level in a BigchainDB config file didn't have any effect. It does now. [Pull request #1797](https://github.com/bigchaindb/bigchaindb/pull/1797)
+* The docs were wrong about there being no Ping/Pong support in the Events API. There is, so the docs were fixed. [Pull request #1799](https://github.com/bigchaindb/bigchaindb/pull/1799)
+* Fixed an issue with closing WebSocket connections properly. [Pull request #1819](https://github.com/bigchaindb/bigchaindb/pull/1819)
+
+### Notes
+* Many changes were made to the Kubernetes-based production deployment template and code.
+
+
+## [1.1] - 2017-09-26
+Tag name: v1.1.0
+
+### Added
+* Support for server-side plugins that can add support for alternate event consumers (other than the WebSocket API). [Pull request #1707](https://github.com/bigchaindb/bigchaindb/pull/1707)
+* New configuration settings to set the *advertised* wsserver scheme, host and port. (The *advertised* ones are the ones that external users use to connect to the WebSocket API.) [Pull request #1703](https://github.com/bigchaindb/bigchaindb/pull/1703)
+* Support for secure (TLS) WebSocket connections. [Pull request #1619](https://github.com/bigchaindb/bigchaindb/pull/1619)
+* A new page of documentation about the contents of a condition (inside a transaction). [Pull request #1668](https://github.com/bigchaindb/bigchaindb/pull/1668)
+
+### Changed
+* We updated our definition of the **public API** (at the top of this document). [Pull request #1700](https://github.com/bigchaindb/bigchaindb/pull/1700)
+* The HTTP API Logger now logs the request path and method as well. [Pull request #1644](https://github.com/bigchaindb/bigchaindb/pull/1644)
+
+### External Contributors
+* @carchrae - [Pull request #1731](https://github.com/bigchaindb/bigchaindb/pull/1731)
+* @ivanbakel - [Pull request #1706](https://github.com/bigchaindb/bigchaindb/pull/1706)
+* @ketanbhatt - Pull requests [#1643](https://github.com/bigchaindb/bigchaindb/pull/1643) and [#1644](https://github.com/bigchaindb/bigchaindb/pull/1644)
+
+### Notes
+* New drivers & tools from our community:
+    * [Java driver](https://github.com/authenteq/java-bigchaindb-driver), by [Authenteq](https://authenteq.com/)
+    * [Ruby library](https://rubygems.org/gems/bigchaindb), by @nileshtrivedi
+* Many improvements to our production deployment template (which uses Kubernetes).
+* The production deployment template for the multi-node case was out of date. We updated that and verified it. [Pull request #1713](https://github.com/bigchaindb/bigchaindb/pull/1713)
+
+
+## [1.0.1] - 2017-07-13
+Tag name: v1.0.1
+
+### Fixed
+* Various issues in the Quickstart page. Pull requests
+ [#1641](https://github.com/bigchaindb/bigchaindb/pull/1641) and
+ [#1648](https://github.com/bigchaindb/bigchaindb/pull/1648).
+* Changefeed hanging when MongoDB primary node is turned off.
+ [Pull request #1638](https://github.com/bigchaindb/bigchaindb/pull/1638).
+* Missing `assets` tables for RethinkDB backend.
+ [Pull request #1646](https://github.com/bigchaindb/bigchaindb/pull/1646).
+* Cryptoconditions version mismatch.
+ [Pull request #1659](https://github.com/bigchaindb/bigchaindb/pull/1659).
+
+
 ## [1.0.0] - 2017-07-05
 Tag name: v1.0.0
 
@@ -42,7 +482,7 @@ Tag name: v1.0.0rc1
 [#1536](https://github.com/bigchaindb/bigchaindb/pull/1536),
 [#1551](https://github.com/bigchaindb/bigchaindb/pull/1551) and
 [#1552](https://github.com/bigchaindb/bigchaindb/pull/1552).
-* Text search support (only if using MongoDB). Pull Requests [#1469](https://github.com/bigchaindb/bigchaindb/pull/1469) and [#1471](https://github.com/bigchaindb/bigchaindb/pull/1471) 
+* Text search support (only if using MongoDB). Pull Requests [#1469](https://github.com/bigchaindb/bigchaindb/pull/1469) and [#1471](https://github.com/bigchaindb/bigchaindb/pull/1471)
 * The `database.connection_timeout` configuration setting now works with RethinkDB too. [#1512](https://github.com/bigchaindb/bigchaindb/pull/1512)
 * New code and tools for benchmarking CREATE transactions. [Pull Request #1511](https://github.com/bigchaindb/bigchaindb/pull/1511)
 
@@ -58,7 +498,7 @@ Tag name: v1.0.0rc1
 * If a node comes back from being down for a while, it will resume voting on blocks in the order determined by the MongoDB oplog, in the case of MongoDB. (In the case of RethinkDB, blocks missed in the changefeed will not be voted on.) [Pull Request #1389](https://github.com/bigchaindb/bigchaindb/pull/1389)
 * Parallelized transaction schema validation in the vote pipeline. [Pull Request #1492](https://github.com/bigchaindb/bigchaindb/pull/1492)
 * `asset.data` or `asset.id` are now *required* in a CREATE or TRANSFER transaction, respectively. [Pull Request #1518](https://github.com/bigchaindb/bigchaindb/pull/1518)
-* The HTTP response body, in the response to the `GET /` and the `GET /api/v1` endpoints, was changed substantially. [Pull Request #1529](https://github.com/bigchaindb/bigchaindb/pull/1529) 
+* The HTTP response body, in the response to the `GET /` and the `GET /api/v1` endpoints, was changed substantially. [Pull Request #1529](https://github.com/bigchaindb/bigchaindb/pull/1529)
 * Changed the HTTP `GET /api/v1/transactions/{transaction_id}` endpoint. It now only returns the transaction if it's in a valid block. It also returns a new header with a relative link to a status monitor. [Pull Request #1543](https://github.com/bigchaindb/bigchaindb/pull/1543)
 * All instances of `txid` and `tx_id` were replaced with `transaction_id`, in the transaction model and the HTTP API. [Pull Request #1532](https://github.com/bigchaindb/bigchaindb/pull/1532)
 * The hostname and port were removed from all URLs in all HTTP API responses. [Pull Request #1538](https://github.com/bigchaindb/bigchaindb/pull/1538)
@@ -98,7 +538,7 @@ Tag name: v0.10.3
 Tag name: v0.10.2
 
 ### Added
-* Add Cross Origin Resource Sharing (CORS) support for the HTTP API. 
+* Add Cross Origin Resource Sharing (CORS) support for the HTTP API.
  [Commit 6cb7596](https://github.com/bigchaindb/bigchaindb/commit/6cb75960b05403c77bdae0fd327612482589efcb)
 
 ### Fixed
@@ -269,7 +709,7 @@ Tag name: v0.8.1
 
 ### Fixed
 - Workaround for rapidjson problem with package metadata extraction
-  (https://github.com/kenrobbins/python-rapidjson/pull/52). 
+  (https://github.com/kenrobbins/python-rapidjson/pull/52).
 
 
 ## [0.8.0] - 2016-11-29
@@ -294,7 +734,7 @@ Tag name: v0.8.0
 - Renamed "verifying key" to "public key". Renamed "signing key" to "private key". Renamed "vk" to "pk". [Pull Request #807](https://github.com/bigchaindb/bigchaindb/pull/807)
 - `get_transaction_by_asset_id` now ignores invalid transactions. [Pull Request #810](https://github.com/bigchaindb/bigchaindb/pull/810)
 - `get_transaction_by_metadata_id` now ignores invalid transactions. [Pull Request #811](https://github.com/bigchaindb/bigchaindb/pull/811)
-- Updates to the configs and scripts for deploying a test network on AWS. The example config file deploys virtual machines running Ubuntu 16.04 now. Pull Requests 
+- Updates to the configs and scripts for deploying a test network on AWS. The example config file deploys virtual machines running Ubuntu 16.04 now. Pull Requests
 [#771](https://github.com/bigchaindb/bigchaindb/pull/771),
 [#813](https://github.com/bigchaindb/bigchaindb/pull/813)
 - Changed logging of transactions on block creation so now it just says the length of the list of transactions, rather than listing all the transactions. [Pull Request #861](https://github.com/bigchaindb/bigchaindb/pull/861)
@@ -362,7 +802,7 @@ committed: Oct 28, 2016, 4:00 PM GMT+2
    2. an `assignee`: the public key of the node it was assigned to.
 - The `assignment_timestamp` wasn't removed before writing the transaction to a block. That was fixed in [Pull Request #627](https://github.com/bigchaindb/bigchaindb/pull/627)
 - The `assignment_timestamp` and `assignee` weren't removed in the response to an HTTP API request sent to the `/api/v1/transactions/<txid>` endpoint. That was fixed in [Pull Request #646](https://github.com/bigchaindb/bigchaindb/pull/646)
-- When validating a TRANSFER transaction, if any fulfillment refers to a transaction that's _not_ in a valid block, then the transaction isn't valid. This wasn't checked before but it is now. [Pull Request #629](https://github.com/bigchaindb/bigchaindb/pull/629)  
+- When validating a TRANSFER transaction, if any fulfillment refers to a transaction that's _not_ in a valid block, then the transaction isn't valid. This wasn't checked before but it is now. [Pull Request #629](https://github.com/bigchaindb/bigchaindb/pull/629)
 
 ### External Contributors
 - @MinchinWeb - [Pull Request #696](https://github.com/bigchaindb/bigchaindb/pull/696)
@@ -390,11 +830,11 @@ committed: Oct 28, 2016, 4:00 PM GMT+2
 [#684](https://github.com/bigchaindb/bigchaindb/pull/684),
 [#688](https://github.com/bigchaindb/bigchaindb/pull/688),
 [#699](https://github.com/bigchaindb/bigchaindb/pull/699),
-[#705](https://github.com/bigchaindb/bigchaindb/pull/705), 
+[#705](https://github.com/bigchaindb/bigchaindb/pull/705),
 [#737](https://github.com/bigchaindb/bigchaindb/pull/737),
-[#748](https://github.com/bigchaindb/bigchaindb/pull/748), 
-[#753](https://github.com/bigchaindb/bigchaindb/pull/753), 
-[#757](https://github.com/bigchaindb/bigchaindb/pull/757), 
+[#748](https://github.com/bigchaindb/bigchaindb/pull/748),
+[#753](https://github.com/bigchaindb/bigchaindb/pull/753),
+[#757](https://github.com/bigchaindb/bigchaindb/pull/757),
 [#759](https://github.com/bigchaindb/bigchaindb/pull/759), and more
 
 
@@ -519,7 +959,7 @@ committed: June 15, 2016, 1:42 PM GMT+2
 
 ### Changed
 - Round timestamps to a precision of one second, and replace payload hash with payload UUID in transactions: [Pull Request #384](https://github.com/bigchaindb/bigchaindb/pull/384)
-- Updated cryptoconditions API usage: [Pull Request #373](https://github.com/bigchaindb/bigchaindb/pull/373) 
+- Updated cryptoconditions API usage: [Pull Request #373](https://github.com/bigchaindb/bigchaindb/pull/373)
 
 
 ## [0.4.1] - 2016-06-13
@@ -564,7 +1004,7 @@ committed: May 27, 2016, 1:42 PM GMT+2
 ### Changed
 - The block processes now use GroupProcess: [Pull Request #267](https://github.com/bigchaindb/bigchaindb/pull/267)
 - Replaced the `json` Python package with `rapidjson` (a Python wrapper for a fast JSON parser/generator written in C++), to speed up JSON serialization and deserialization: [Pull Request #318](https://github.com/bigchaindb/bigchaindb/pull/318)
-- Overhauled `ROADMAP.md` and moved it to [the bigchaindb/org repository](https://github.com/bigchaindb/org): Pull Requests 
+- Overhauled `ROADMAP.md` and moved it to [the bigchaindb/org repository](https://github.com/bigchaindb/org): Pull Requests
 [#282](https://github.com/bigchaindb/bigchaindb/pull/282),
 [#306](https://github.com/bigchaindb/bigchaindb/pull/306),
 [#308](https://github.com/bigchaindb/bigchaindb/pull/308),
@@ -612,18 +1052,18 @@ committed: April 26, 2016, 11:09 AM GMT+2
 - Ability to use environment variables to set (or partially set) configuration settings: [Pull Request #153](https://github.com/bigchaindb/bigchaindb/pull/153)
 - `bigchaindb --export-my-pubkey`: [Pull Request #186](https://github.com/bigchaindb/bigchaindb/pull/186)
 - `bigchaindb --version`, and one central source for the current version (`version.py`): [Pull Request #208](https://github.com/bigchaindb/bigchaindb/pull/208)
-- AWS deployment scripts: Pull Requests 
+- AWS deployment scripts: Pull Requests
 [#160](https://github.com/bigchaindb/bigchaindb/pull/160),
 [#166](https://github.com/bigchaindb/bigchaindb/pull/166),
 [#172](https://github.com/bigchaindb/bigchaindb/pull/172),
 [#203](https://github.com/bigchaindb/bigchaindb/pull/203)
 - `codecov.yml`: [Pull Request #161](https://github.com/bigchaindb/bigchaindb/pull/161)
 - `CHANGELOG.md` (this file): [Pull Request #117](https://github.com/bigchaindb/bigchaindb/pull/117)
-- Signatures using Ed25519: Pull Requests 
+- Signatures using Ed25519: Pull Requests
 [#138](https://github.com/bigchaindb/bigchaindb/pull/138),
 [#152](https://github.com/bigchaindb/bigchaindb/pull/152)
 - Multisig support: [Pull Request #107](https://github.com/bigchaindb/bigchaindb/pull/107)
-- HTTP Server & Web API: Pull Requests 
+- HTTP Server & Web API: Pull Requests
 [#102](https://github.com/bigchaindb/bigchaindb/pull/102),
 [#150](https://github.com/bigchaindb/bigchaindb/pull/150),
 [#155](https://github.com/bigchaindb/bigchaindb/pull/155),
@@ -642,7 +1082,7 @@ committed: April 26, 2016, 11:09 AM GMT+2
 - Bug related to running the `bigchaindb-benchmark load` on docker [Pull Request #225](https://github.com/bigchaindb/bigchaindb/pull/225)
 
 ## External Contributors
-- [@thedoctor](https://github.com/thedoctor): Pull Requests 
+- [@thedoctor](https://github.com/thedoctor): Pull Requests
 [#99](https://github.com/bigchaindb/bigchaindb/pull/99),
 [#136](https://github.com/bigchaindb/bigchaindb/pull/136)
 - [@roderik](https://github.com/roderik): [Pull Request #162](https://github.com/bigchaindb/bigchaindb/pull/162)
@@ -658,8 +1098,8 @@ committed: April 20, 2016, 3:31 PM GMT+2
 
 
 ## [0.1.4] - 2016-02-22
-Tag name: v0.1.4 
-= commit: c4c850f480bc9ae72df2a54f81c0825b6fb4ed62 
+Tag name: v0.1.4
+= commit: c4c850f480bc9ae72df2a54f81c0825b6fb4ed62
 committed: Feb 22, 2016, 11:51 AM GMT+1
 
 ### Added
@@ -680,8 +1120,8 @@ committed Feb 16, 2016, 11:37 AM GMT+1
 
 
 ## [0.1.2] - 2016-02-15
-Tag name: v0.1.2 
-= commit d2ff24166d69dda68dd7b4a24a88279b1d37e222 
+Tag name: v0.1.2
+= commit d2ff24166d69dda68dd7b4a24a88279b1d37e222
 committed Feb 15, 2016, 2:23 PM GMT+1
 
 ### Added
@@ -691,8 +1131,8 @@ committed Feb 15, 2016, 2:23 PM GMT+1
 - Fix exception when running `start`: [Pull Request #32](https://github.com/bigchaindb/bigchaindb/pull/32) resolved [Issue #35]
 
 ## [0.1.1] - 2016-02-15
-Tag name: v0.1.1 
-= commit 2a025448b29fe7056760de1039c73bbcfe992461 
+Tag name: v0.1.1
+= commit 2a025448b29fe7056760de1039c73bbcfe992461
 committed Feb 15, 2016, 10:48 AM GMT+1
 
 ### Added

CODE_OF_CONDUCT.md

@@ -1,3 +1,10 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Contributor Code of Conduct
 
 As contributors and maintainers of this project, and in the interest of
@@ -35,7 +42,7 @@ This Code of Conduct applies both within project spaces and in public spaces
 when an individual is representing the project or its community.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior directed at yourself or another community member may be
-reported by contacting a project maintainer at [conduct@bigchaindb.com](mailto:conduct@bigchaindb.com). All
+reported by contacting a project maintainer at [contact@bigchaindb.com](mailto:contact@bigchaindb.com). All
 complaints will be reviewed and investigated and will result in a response that
 is appropriate to the circumstances. Maintainers are
 obligated to maintain confidentiality with regard to the reporter of an

CONTRIBUTING.md

@@ -1,163 +0,0 @@
-# How to Contribute to the BigchainDB Project
-
-There are many ways you can contribute to the BigchainDB project, some very easy and others more involved. We want to be friendly and welcoming to all potential contributors, so we ask that everyone involved abide by some simple guidelines outlined in our [Code of Conduct](./CODE_OF_CONDUCT.md).
-
-Or, are you interested in contributing full-time? BigchainDB is hiring. See [here](https://github.com/bigchaindb/org/blob/master/engjob.md).
-
-## Easy Ways to Contribute
-
-The BigchainDB community has a Google Group and a Gitter chatroom. Our [Community page](https://www.bigchaindb.com/community) has more information about those.
-
-You can also follow us on Twitter [@BigchainDB](https://twitter.com/BigchainDB) or read [our blog on Medium](https://medium.com/the-bigchaindb-blog).
-
-If you want to file a bug report, suggest a feature, or ask a code-related question, please go to the `bigchaindb/bigchaindb` repository on GitHub and [create a new Issue](https://github.com/bigchaindb/bigchaindb/issues/new). (You will need a [GitHub account](https://github.com/signup/free) (free).) Please describe the issue clearly, including steps to reproduce when it is a bug.
-
-## How to Contribute Code or Documentation
-
-### Step 0 - Decide on an Issue to Resolve, or Create One
-
-We want you to feel like your contributions (pull requests) are welcome, but if you contribute something unnecessary, unwanted, or perplexing, then your experience may be unpleasant. Your pull request may sit gathering dust as everyone scratches their heads wondering what to do with it.
-
-To prevent that situation, we ask that all pull requests should resolve, address, or fix an existing issue. If there is no existing issue, then you should create one first. That way there can be commentary and discussion first, and you can have a better idea of what to expect when you create a corresponding pull request.
-
-When you submit a pull request, please mention the issue (or issues) that it resolves, e.g. "Resolves #123".
-
-Exception: hotfixes and minor changes don't require a pre-existing issue, but please write a thorough pull request description.
-
-### Step 1 - Prepare and Familiarize Yourself
-
-To contribute code or documentation, you need a [GitHub account](https://github.com/signup/free).
-
-Familiarize yourself with how we do coding and documentation in the BigchainDB project, including:
-
-* [our Python Style Guide](PYTHON_STYLE_GUIDE.md)
-* [how we write and run tests](./tests/README.md)
-* [our documentation strategy](./docs/README.md) (including in-code documentation)
-* the GitHub Flow (workflow)
-    * [GitHub Guide: Understanding the GitHub Flow](https://guides.github.com/introduction/flow/)
-    * [Scott Chacon's blog post about GitHub Flow](http://scottchacon.com/2011/08/31/github-flow.html)
-* [semantic versioning](http://semver.org/)
-
-### Step 2 - Install some Dependencies
-
-* [Install RethinkDB Server](https://rethinkdb.com/docs/install/)
-* Make sure you have Python 3.5+ (preferably in a virtualenv)
-* [Install BigchaindB Server's OS-level dependencies](https://docs.bigchaindb.com/projects/server/en/latest/appendices/install-os-level-deps.html)
-* [Make sure you have the latest Python 3 version of pip and setuptools](https://docs.bigchaindb.com/projects/server/en/latest/appendices/install-latest-pip.html)
-
-### Step 3 - Fork bigchaindb on GitHub
-
-In your web browser, go to [the BigchainDB repository on GitHub](https://github.com/bigchaindb/bigchaindb) and click the `Fork` button in the top right corner. This creates a new Git repository named `bigchaindb` in _your_ GitHub account.
-
-### Step 4 - Clone Your Fork
-
-(This only has to be done once.) In your local terminal, use Git to clone _your_ `bigchaindb` repository to your local computer. Also add the original GitHub bigchaindb/bigchaindb repository as a remote named `upstream` (a convention):
-```text
-git clone git@github.com:your-github-username/bigchaindb.git
-cd bigchaindb
-git remote add upstream git@github.com:bigchaindb/bigchaindb.git
-```
-
-### Step 5 - Fetch and Merge the Latest from `upstream/master`
-
-Switch to the `master` branch locally, fetch all `upstream` branches, and merge the just-fetched `upstream/master` branch with the local `master` branch:
-```text
-git checkout master
-git fetch upstream
-git merge upstream/master
-```
-
-### Step 6 - Install the Python module and the CLI
-
-In order to use and run the source you just cloned from your fork, you need to install BigchainDB on your computer.
-The core of BigchainDB is a Python module you can install using the standard [Python packaging tools](http://python-packaging-user-guide.readthedocs.org/en/latest/).
-We highly suggest you use `pip` and `virtualenv` to manage your local development.
-If you need more information on how to do that, refer to the *Python Packaging User Guide* to [install `pip`](http://python-packaging-user-guide.readthedocs.org/en/latest/installing/#requirements-for-installing-packages) and to [create your first `virtualenv`](http://python-packaging-user-guide.readthedocs.org/en/latest/installing/#creating-virtual-environments).
-
-Once you have `pip` installed and (optionally) you are in a virtualenv, go to the root of the repository (i.e. where the `setup.py` file is), and type:
-```text
-pip install -e .[dev]
-```
-
-This will install the BigchainDB Python module, the CLI, and all the dependencies useful for contributing to the development of BigchainDB.
-How? Let's split the command down into its components:
- - `pip` is the Python command to install packages
- - `install` tells pip to use the *install* action
- - `-e` installs a project in [editable mode](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)
- - `.` installs what's in the current directory
- - `[dev]` adds some [extra requirements](https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies) to the installation. (If you are curious, open `setup.py` and look for `dev` in the `extras_require` section.)
-
-Aside: An alternative to `pip install -e .[dev]` is `python setup.py develop`.
-
-
-### Step 7 - Create a New Branch for Each Bug/Feature
-
-If your new branch is to **fix a bug** identified in a specific GitHub Issue with number `ISSNO`, then name your new branch `bug/ISSNO/short-description-here`. For example, `bug/67/fix-leap-year-crash`.
-
-If your new branch is to **add a feature** requested in a specific GitHub Issue with number `ISSNO`, then name your new branch `feat/ISSNO/short-description-here`. For example, `feat/135/blue-background-on-mondays`.
-
-Otherwise, please give your new branch a short, descriptive, all-lowercase name.
-```text
-git checkout -b new-branch-name
-```
-
-### Step 8 - Make Edits, git add, git commit
-
-With your new branch checked out locally, make changes or additions to the code or documentation. Remember to:
-
-* follow [our Python Style Guide](PYTHON_STYLE_GUIDE.md).
-* write and run tests for any new or changed code. There's [a README file in the `tests/` folder](./tests/README.md) about how to do that.
-* add or update documentation as necessary. Follow [our documentation strategy](./docs/README.md).
-
-As you go, git add and git commit your changes or additions, e.g.
-```text
-git add new-or-changed-file-1
-git add new-or-changed-file-2
-git commit -m "Short description of new or changed things"
-```
-
-You will want to merge changes from upstream (i.e. the original repository) into your new branch from time to time, using something like:
-```text
-git fetch upstream
-git merge upstream/master
-```
-
-Once you're done commiting a set of new things and you're ready to submit them for inclusion, please be sure to run all the tests as per the instructions in [the README file in the `tests/` folder](./tests/README.md).
-
-(When you submit your pull request [following the instructions below], we run all the tests automatically, so we will see if some are failing. If you don't know why some tests are failing, you can still submit your pull request, but be sure to note the failing tests and to ask for help with resolving them.)
-
-### Step 9 - Push Your New Branch to origin
-
-Make sure you've commited all the additions or changes you want to include in your pull request. Then push your new branch to origin (i.e. _your_ remote bigchaindb repository).
-```text
-git push origin new-branch-name
-```
-
-### Step 10 - Create a Pull Request 
-
-Go to the GitHub website and to _your_ remote bigchaindb repository (i.e. something like https://github.com/your-user-name/bigchaindb). 
-
-See [GitHub's documentation on how to initiate and send a pull request](https://help.github.com/articles/using-pull-requests/). Note that the destination repository should be `bigchaindb/bigchaindb` and the destination branch will be `master` (usually, and if it's not, then we can change that if necessary).
-
-If this is the first time you've submitted a pull request to BigchainDB, then you must read and accept the Contributor License Agreement (CLA) before we can merge your contributions. That can be found at [https://www.bigchaindb.com/cla](https://www.bigchaindb.com/cla).
-
-Once you accept and submit the CLA, we'll email you with further instructions. (We will send you a long random string to put in the comments section of your pull request, along with the text, "I have read and agree to the terms of the BigchainDB Contributor License Agreement.")
-
-Someone will then merge your branch or suggest changes. If we suggest changes, you won't have to open a new pull request, you can just push new code to the same branch (on `origin`) as you did before creating the pull request.
-
-### Tip: Upgrading All BigchainDB Dependencies
-
-Over time, your versions of the Python packages used by BigchainDB will get out of date. You can upgrade them using:
-```text
-pip install --upgrade -e .[dev]
-```
-
-## Quick Links
-
-* [BigchainDB Community links](https://www.bigchaindb.com/community)
-* [General GitHub Documentation](https://help.github.com/)
-* [Code of Conduct](./CODE_OF_CONDUCT.md)
-* [BigchainDB Licenses](./LICENSES.md)
-* [Contributor License Agreement](https://www.bigchaindb.com/cla)
-
-(Note: GitHub automatically links to CONTRIBUTING.md when a contributor creates an Issue or opens a Pull Request.)

Dockerfile

@@ -1,18 +1,24 @@
 FROM python:3.6
-LABEL maintainer "dev@bigchaindb.com"
+LABEL maintainer "contact@ipdb.global"
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app/
 WORKDIR /usr/src/app
 RUN apt-get -qq update \
     && apt-get -y upgrade \
-    && pip install --no-cache-dir . \
+    && apt-get install -y jq \
+    && pip install . \
     && apt-get autoremove \
     && apt-get clean
+
 VOLUME ["/data", "/certs"]
-WORKDIR /data
+
+ENV PYTHONUNBUFFERED 0
 ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb
 ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
 ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
 ENV BIGCHAINDB_WSSERVER_SCHEME ws
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_PORT 9985
 ENTRYPOINT ["bigchaindb"]
 CMD ["start"]

Dockerfile-all-in-one

@@ -0,0 +1,51 @@
+FROM alpine:3.9
+LABEL maintainer "contact@ipdb.global"
+
+ARG TM_VERSION=v0.31.5
+RUN mkdir -p /usr/src/app
+ENV HOME /root
+COPY . /usr/src/app/
+WORKDIR /usr/src/app
+
+RUN apk --update add sudo bash \
+    && apk --update add python3 openssl ca-certificates git \
+    && apk --update add --virtual build-dependencies python3-dev \
+        libffi-dev openssl-dev build-base jq \
+    && apk add --no-cache libstdc++ dpkg gnupg \
+    && pip3 install --upgrade pip cffi \
+    && pip install -e . \
+    && apk del build-dependencies \
+    && rm -f /var/cache/apk/*
+
+# Install mongodb and monit
+RUN apk --update add mongodb monit
+
+# Install Tendermint
+RUN wget https://github.com/tendermint/tendermint/releases/download/${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip \
+    && unzip tendermint_${TM_VERSION}_linux_amd64.zip \
+    && mv tendermint /usr/local/bin/ \
+    && rm tendermint_${TM_VERSION}_linux_amd64.zip
+
+ENV TMHOME=/tendermint
+
+# Set permissions required for mongodb
+RUN mkdir -p /data/db /data/configdb \
+        && chown -R mongodb:mongodb /data/db /data/configdb
+
+# BigchainDB enviroment variables
+ENV BIGCHAINDB_DATABASE_PORT 27017
+ENV BIGCHAINDB_DATABASE_BACKEND localmongodb
+ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
+ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_SCHEME ws
+
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
+ENV BIGCHAINDB_TENDERMINT_PORT 26657
+
+VOLUME /data/db /data/configdb /tendermint
+
+EXPOSE 27017 28017 9984 9985 26656 26657 26658
+
+WORKDIR $HOME
+ENTRYPOINT ["/usr/src/app/pkg/scripts/all-in-one.bash"]

Dockerfile-alpine

@@ -0,0 +1,30 @@
+FROM alpine:latest
+LABEL maintainer "contact@ipdb.global"
+RUN mkdir -p /usr/src/app
+COPY . /usr/src/app/
+WORKDIR /usr/src/app
+RUN apk --update add sudo \
+    && apk --update add python3 py-pip openssl ca-certificates git\
+    && apk --update add --virtual build-dependencies python3-dev \
+        libffi-dev openssl-dev build-base \
+    && apk add --no-cache libstdc++ \
+    && pip3 install --upgrade pip cffi \
+    && pip install -e . \
+    && apk del build-dependencies \
+    && rm -f /var/cache/apk/*
+# When developing with Python in a docker container, we are using PYTHONBUFFERED
+# to force stdin, stdout and stderr to be totally unbuffered and to capture logs/outputs
+ENV PYTHONUNBUFFERED 0
+
+ENV BIGCHAINDB_DATABASE_PORT 27017
+ENV BIGCHAINDB_DATABASE_BACKEND $backend
+ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
+ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_SCHEME ws
+
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
+
+ENV BIGCHAINDB_TENDERMINT_PORT 26657
+ARG backend
+RUN bigchaindb -y configure "$backend"

Dockerfile-dev

@@ -1,21 +1,35 @@
-FROM python:3.6
-LABEL maintainer "dev@bigchaindb.com"
+ARG python_version=3.6
+FROM python:${python_version}
+LABEL maintainer "contact@ipdb.global"
 
 RUN apt-get update \
-    && apt-get install -y vim \
+    && apt-get install -y git \
     && pip install -U pip \
-    && pip install pynacl \
     && apt-get autoremove \
     && apt-get clean
 
+ARG backend
+ARG abci_status
+
+# When developing with Python in a docker container, we are using PYTHONBUFFERED
+# to force stdin, stdout and stderr to be totally unbuffered and to capture logs/outputs
+ENV PYTHONUNBUFFERED 0
+
+ENV BIGCHAINDB_DATABASE_PORT 27017
+ENV BIGCHAINDB_DATABASE_BACKEND $backend
 ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
 ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
 ENV BIGCHAINDB_WSSERVER_SCHEME ws
 
-ARG backend
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
+
+ENV BIGCHAINDB_TENDERMINT_PORT 26657
+
+ENV BIGCHAINDB_CI_ABCI ${abci_status}
 
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app/
 WORKDIR /usr/src/app
-RUN pip install --no-cache-dir -e .[dev]
-RUN bigchaindb -y configure "$backend"
+RUN pip install -e .[dev]
+RUN bigchaindb -y configure

HOW_TO_HANDLE_PULL_REQUESTS.md

@@ -1,65 +0,0 @@
-# How to Handle Pull Requests
-
-This document is for whoever has the ability to merge pull requests in the Git repositories associated with BigchainDB.
-
-If the pull request is from an employee of BigchainDB GmbH, then you can ignore this document.
-
-If the pull request is from someone who is _not_ an employee of BigchainDB, then:
-
-* Have they agreed to the Individual Contributor Agreement in the past? (Troy, Greg, and others have a list.) If yes, then you can merge the PR and ignore the rest of this document.
-* Do they belong to a company or organization which agreed to the Entity Contributor Agreement in the past, and will they be contributing on behalf of that company or organization? (Troy, Greg, and others have a list.) If yes, then you can merge the PR and ignore the rest of this document.
-* Otherwise, go to the pull request in question and post a comment using this template:
-
-Hi @nameofuser
-
-Before we can merge this pull request, we need you or your organization to agree to one of our contributor agreements. One of the big concerns for people using and developing open source software is that someone who contributed to the code might claim the code infringes on their copyright or patent. To guard against this, we ask all our contributors to sign a Contributor License Agreement. This gives us the right to use the code contributed and any patents the contribution relies on. It also gives us and our users comfort that they won't be sued for using open source software. We know it's a hassle, but it makes the project more reliable in the long run. Thank you for your understanding and your contribution!
-
-If you are contributing on behalf of yourself (and not on behalf of your employer or another organization you are part of) then you should:
-
-1. Go to: https://www.bigchaindb.com/cla/
-2. Read the Individual Contributor Agreement
-3. Fill in the form "For Individuals"
-4. Check the box to agree
-5. Click the SEND button
-
-If you're contributing as an employee, and/or you want all employees of your employing organization to be covered by our contributor agreement, then someone in your organization with the authority to enter agreements on behalf of all employees must do the following:
-
-1. Go to: https://www.bigchaindb.com/cla/
-2. Read the Entity Contributor Agreement
-3. Fill in the form "For Organizations”
-4. Check the box to agree
-5. Click the SEND button
-
-We will email you (or your employer) with further instructions.
-
-(END OF COMMENT)
-
-Once they click SEND, we (BigchainDB) will get an email with the information in the form. (Troy gets those emails for sure, I'm not sure who else.) The next step is to send an email to the email address submitted in the form, saying something like (where the stuff in [square brackets] should be replaced):
-
-Hi [NAME],
-
-The next step is for you to copy the following block of text into the comments of Pull Request #[NN] on GitHub:
-
-BEGIN BLOCK
-
-This is to confirm that I agreed to and accepted the BigchainDB [Entity/Individual] Contributor Agreement at https://www.bigchaindb.com/cla/ and to represent and warrant that I have authority to do so.
-
-[Insert long random string here. One good source of those is https://www.grc.com/passwords.htm ]
-
-END BLOCK
-
-(END OF EMAIL)
-
-The next step is to wait for them to copy that comment into the comments of the indicated pull request. Once they do so, it's safe to merge the pull request.
-
-## How to Handle CLA Agreement Emails with No Associated Pull Request
-
-Reply with an email like this:
-
-Hi [First Name],
-
-Today I got an email (copied below) to tell me that you agreed to the BigchainDB Contributor License Agreement. Did you intend to do that?
-
-If no, then you can ignore this email.
-
-If yes, then there's another step to connect your email address with your GitHub account. To do that, you must first create a pull request in one of the BigchainDB repositories on GitHub. Once you've done that, please reply to this email with a link to the pull request. Then I'll send you a special block of text to paste into the comments on that pull request.

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSES.md

@@ -1,23 +1,24 @@
-# Code Licenses
+# Copyrights and Licenses
 
-For all code in this repository, BigchainDB GmbH ("We") either:
+## Copyrights
 
-1. owns the copyright, or
-2. owns the right to sublicense it (because all external contributors must agree to a Contributor License Agreement).
+For all the code and documentation in this repository, the copyright is owned by one or more of the following:
 
-Therefore We can choose how to license all the code in this repository. We can license it to Joe Xname under one license and Company Yname under a different license.
+- BigchainDB GmbH
+- A BigchainDB contributor who agreed to a BigchainDB Contributor License Agreement (CLA) with BigchainDB GmbH. (See [BEP-16](https://github.com/bigchaindb/BEPs/tree/master/16).)
+- A BigchainDB contributor who signed off on the Developer Certificate of Origin (DCO) for all their contributions. (See [BEP-24](https://github.com/bigchaindb/BEPs/tree/master/24).)
+- (Rarely, see the **Exceptions Section** below) A third pary who licensed the code in question under an open source license.
 
-The two general options are:
+## Code Licenses
 
-1. You can get it under a commercial license for a fee. We can negotiate the terms of that license. It's not like we have some standard take-it-or-leave it commercial license. If you want to modify it and keep your modifications private, then that's certainly possible. Just ask.
-2. You can get it under the AGPLv3 license for free. You don't even have to ask us. That's because all code in _this_ repository is licensed under the GNU Affero General Public License version 3 (AGPLv3), the full text of which can be found at [http://www.gnu.org/licenses/agpl.html](http://www.gnu.org/licenses/agpl.html).
+All code in this repository, including short code snippets in the documentation, but not including the **Exceptions** noted below, is licensed under the Apache License, Version 2.0, the full text of which can be found at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0).
 
-If you don't like the AGPL license, then just ignore it. It doesn't affect any other license.
+For the licenses on all other BigchainDB-related code (i.e. in other repositories), see the LICENSE file in the associated repository.
 
-All short code snippets embedded in the official BigchainDB _documentation_ are also licensed under the Apache License, Version 2.0, the full text of which can be found at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0).
+## Documentation Licenses
 
-For the licenses on all other BigchainDB-related code, see the LICENSE file in the associated repository.
+The official BigchainDB documentation, _except for the short code snippets embedded within it_, is licensed under a Creative Commons Attribution 4.0 International license, the full text of which can be found at [http://creativecommons.org/licenses/by/4.0/legalcode](http://creativecommons.org/licenses/by/4.0/legalcode).
 
-# Documentation Licenses
+## Exceptions
 
-The official BigchainDB documentation, _except for the short code snippets embedded within it_, is licensed under a Creative Commons Attribution-ShareAlike 4.0 International license, the full text of which can be found at [http://creativecommons.org/licenses/by-sa/4.0/legalcode](http://creativecommons.org/licenses/by-sa/4.0/legalcode).
+The contents of the `k8s/nginx-openresty/` directory are licensed as described in the `LICENSE.md` file in that directory.

Makefile

@@ -1,5 +1,11 @@
-.PHONY: clean clean-test clean-pyc clean-build docs help
+.PHONY: help run start stop logs test test-unit test-unit-watch test-acceptance cov doc doc-acceptance clean reset release dist check-deps clean-build clean-pyc clean-test
+
 .DEFAULT_GOAL := help
+
+
+#############################
+# Open a URL in the browser #
+#############################
 define BROWSER_PYSCRIPT
 import os, webbrowser, sys
 try:
@@ -11,64 +17,88 @@ webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
 endef
 export BROWSER_PYSCRIPT
 
+
+##################################
+# Display help for this makefile #
+##################################
 define PRINT_HELP_PYSCRIPT
 import re, sys
 
+print("BigchainDB 2.0 developer toolbox")
+print("--------------------------------")
+print("Usage:  make COMMAND")
+print("")
+print("Commands:")
 for line in sys.stdin:
 	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
 	if match:
 		target, help = match.groups()
-		print("%-20s %s" % (target, help))
+		print("    %-16s %s" % (target, help))
 endef
 export PRINT_HELP_PYSCRIPT
+
+##################
+# Basic commands #
+##################
+DOCKER := docker
+DC := docker-compose
 BROWSER := python -c "$$BROWSER_PYSCRIPT"
+HELP := python -c "$$PRINT_HELP_PYSCRIPT"
+ECHO := /usr/bin/env echo
 
-help:
-	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+IS_DOCKER_COMPOSE_INSTALLED := $(shell command -v docker-compose 2> /dev/null)
 
-clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+################
+# Main targets #
+################
 
+help: ## Show this help
+	@$(HELP) < $(MAKEFILE_LIST)
 
-clean-build: ## remove build artifacts
-	rm -fr build/
-	rm -fr dist/
-	rm -fr .eggs/
-	find . -name '*.egg-info' -exec rm -fr {} +
-	find . -name '*.egg' -exec rm -f {} +
+run: check-deps ## Run BigchainDB from source (stop it with ctrl+c)
+	# although bigchaindb has tendermint and mongodb in depends_on,
+	# launch them first otherwise tendermint will get stuck upon sending yet another log
+	# due to some docker-compose issue; does not happen when containers are run as daemons
+	@$(DC) up --no-deps mongodb tendermint bigchaindb
 
-clean-pyc: ## remove Python file artifacts
-	find . -name '*.pyc' -exec rm -f {} +
-	find . -name '*.pyo' -exec rm -f {} +
-	find . -name '*~' -exec rm -f {} +
-	find . -name '__pycache__' -exec rm -fr {} +
+start: check-deps ## Run BigchainDB from source and daemonize it (stop with `make stop`)
+	@$(DC) up -d bigchaindb
 
-clean-test: ## remove test and coverage artifacts
-	rm -fr .tox/
-	rm -f .coverage
-	rm -fr htmlcov/
+stop: check-deps ## Stop BigchainDB
+	@$(DC) stop
 
-lint: ## check style with flake8
-	flake8 bigchaindb tests
+logs: check-deps ## Attach to the logs
+	@$(DC) logs -f bigchaindb
 
-test: ## run tests quickly with the default Python
-	pytest -v -n auto
+test: check-deps test-unit test-acceptance ## Run unit and acceptance tests
 
-test-all: ## run tests on every Python version with tox
-	tox
+test-unit: check-deps ## Run all tests once
+	@$(DC) up -d bdb
+	@$(DC) exec bigchaindb pytest
 
-coverage: ## check code coverage quickly with the default Python
-	pytest -v -n auto --cov=bigchaindb --cov-report term --cov-report html
+test-unit-watch: check-deps ## Run all tests and wait. Every time you change code, tests will be run again
+	@$(DC) run --rm --no-deps bigchaindb pytest -f
+
+test-acceptance: check-deps ## Run all acceptance tests
+	@./run-acceptance-test.sh
+
+cov: check-deps ## Check code coverage and open the result in the browser
+	@$(DC) run --rm bigchaindb pytest -v --cov=bigchaindb --cov-report html
 	$(BROWSER) htmlcov/index.html
 
-docs: ## generate Sphinx HTML documentation, including API docs
-	$(MAKE) -C docs/root clean
-	$(MAKE) -C docs/root html
-	$(MAKE) -C docs/server clean
-	$(MAKE) -C docs/server html
-	$(BROWSER) docs/root/_build/html/index.html
+doc: check-deps ## Generate HTML documentation and open it in the browser
+	@$(DC) run --rm --no-deps bdocs make -C docs/root html
+	$(BROWSER) docs/root/build/html/index.html
 
-servedocs: docs ## compile the docs watching for changes
-	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
+doc-acceptance: check-deps ## Create documentation for acceptance tests
+	@$(DC) run --rm python-acceptance pycco -i -s /src -d /docs
+	$(BROWSER) acceptance/python/docs/index.html
+
+clean: clean-build clean-pyc clean-test ## Remove all build, test, coverage and Python artifacts
+	@$(ECHO) "Cleaning was successful."
+
+reset: check-deps ## Stop and REMOVE all containers. WARNING: you will LOSE all data stored in BigchainDB.
+	@$(DC) down
 
 release: dist ## package and upload a release
 	twine upload dist/*
@@ -78,5 +108,35 @@ dist: clean ## builds source (and not for now, wheel package)
 	# python setup.py bdist_wheel
 	ls -l dist
 
-install: clean ## install the package to the active Python's site-packages
-	python setup.py install
+###############
+# Sub targets #
+###############
+
+check-deps:
+ifndef IS_DOCKER_COMPOSE_INSTALLED
+	@$(ECHO) "Error: docker-compose is not installed"
+	@$(ECHO)
+	@$(ECHO) "You need docker-compose to run this command. Check out the official docs on how to install it in your system:"
+	@$(ECHO) "- https://docs.docker.com/compose/install/"
+	@$(ECHO)
+	@$(DC) # docker-compose is not installed, so we call it to generate an error and exit
+endif
+
+clean-build: # Remove build artifacts
+	@rm -fr build/
+	@rm -fr dist/
+	@rm -fr .eggs/
+	@find . -name '*.egg-info' -exec rm -fr {} +
+	@find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: # Remove Python file artifacts
+	@find . -name '*.pyc' -exec rm -f {} +
+	@find . -name '*.pyo' -exec rm -f {} +
+	@find . -name '*~' -exec rm -f {} +
+	@find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: # Remove test and coverage artifacts
+	@find . -name '.pytest_cache' -exec rm -fr {} +
+	@rm -fr .tox/
+	@rm -f .coverage
+	@rm -fr htmlcov/

PYTHON_STYLE_GUIDE.md

@@ -1,3 +1,10 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Python Style Guide
 
 This guide starts out with our general Python coding style guidelines and ends with a section on how we write & run (Python) tests.
@@ -8,6 +15,9 @@ Our starting point is [PEP8](https://www.python.org/dev/peps/pep-0008/), the sta
 
 BigchainDB uses Python 3.5+, so you can ignore all PEP8 guidelines specific to Python 2.
 
+We use [pre-commit](http://pre-commit.com/) to check some of the rules below before every commit but not everything is realized yet.
+The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file.
+
 ### Python Docstrings
 
 PEP8 says some things about docstrings, but not what to put in them or how to structure them. [PEP257](https://www.python.org/dev/peps/pep-0257/) was one proposal for docstring conventions, but we prefer [Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) instead: they're easier to read and the [napoleon extension](http://www.sphinx-doc.org/en/stable/ext/napoleon.html) for Sphinx lets us turn them into nice-looking documentation. Here are some references on Google-style docstrings:
@@ -82,6 +92,6 @@ flake8 --max-line-length 119 bigchaindb/
 
 ## Writing and Running (Python) Tests
 
-The content of this section was moved to [`bigchaindb/tests/README.md`](./tests/README.md).
+The content of this section was moved to [`bigchaindb/tests/README.md`](https://github.com/bigchaindb/bigchaindb/blob/master/tests/README.md).
 
 Note: We automatically run all tests on all pull requests (using Travis CI), so you should definitely run all tests locally before you submit a pull request. See the above-linked README file for instructions.

README.md

@@ -1,25 +1,61 @@
-[![PyPI](https://img.shields.io/pypi/status/bigchaindb.svg?maxAge=2592000)](https://pypi.python.org/pypi/BigchainDB)
-[![PyPI](https://img.shields.io/pypi/v/bigchaindb.svg)](https://pypi.python.org/pypi/BigchainDB)
-[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.org/bigchaindb/bigchaindb)
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
+<!--- There is no shield to get the latest version
+(including pre-release versions) from PyPI,
+so show the latest GitHub release instead.
+--->
+
 [![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master)
+[![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases)
+[![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/BigchainDB/)
+[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.com/bigchaindb/bigchaindb)
 [![Documentation Status](https://readthedocs.org/projects/bigchaindb-server/badge/?version=latest)](https://docs.bigchaindb.com/projects/server/en/latest/)
 [![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-
 # BigchainDB Server
 
-BigchainDB is a scalable blockchain database. [The whitepaper](https://www.bigchaindb.com/whitepaper/) explains what that means.
+BigchainDB is the blockchain database. This repository is for _BigchainDB Server_.
 
-## Get Started with BigchainDB Server
+## The Basics
 
-### [Quickstart](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html)
-### [Set Up & Run a Dev/Test Node](https://docs.bigchaindb.com/projects/server/en/latest/dev-and-test/setup-run-node.html)
-### [Run BigchainDB Server with Docker](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-docker.html)
+* [Try the Quickstart](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html)
+* [Read the BigchainDB 2.0 whitepaper](https://www.bigchaindb.com/whitepaper/)
+* [Check out the _Hitchiker's Guide to BigchainDB_](https://www.bigchaindb.com/developers/guide/)
+
+## Run and Test BigchainDB Server from the `master` Branch
+
+Running and testing the latest version of BigchainDB Server is easy. Make sure you have a recent version of [Docker Compose](https://docs.docker.com/compose/install/) installed. When you are ready, fire up a terminal and run:
+
+```text
+git clone https://github.com/bigchaindb/bigchaindb.git
+cd bigchaindb
+make run
+```
+
+BigchainDB should be reachable now on `http://localhost:9984/`.
+
+There are also other commands you can execute:
+
+* `make start`: Run BigchainDB from source and daemonize it (stop it with `make stop`).
+* `make stop`: Stop BigchainDB.
+* `make logs`: Attach to the logs.
+* `make test`: Run all unit and acceptance tests.
+* `make test-unit-watch`: Run all tests and wait. Every time you change code, tests will be run again.
+* `make cov`: Check code coverage and open the result in the browser.
+* `make doc`: Generate HTML documentation and open it in the browser.
+* `make clean`: Remove all build, test, coverage and Python artifacts.
+* `make reset`: Stop and REMOVE all containers. WARNING: you will LOSE all data stored in BigchainDB.
+
+To view all commands available, run `make`.
 
 ## Links for Everyone
 
 * [BigchainDB.com](https://www.bigchaindb.com/) - the main BigchainDB website, including newsletter signup
-* [Whitepaper](https://www.bigchaindb.com/whitepaper/) - outlines the motivations, goals and core algorithms of BigchainDB
 * [Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md)
 * [Blog](https://medium.com/the-bigchaindb-blog)
 * [Twitter](https://twitter.com/BigchainDB)
@@ -28,7 +64,7 @@ BigchainDB is a scalable blockchain database. [The whitepaper](https://www.bigch
 
 * [All BigchainDB Documentation](https://docs.bigchaindb.com/en/latest/)
 * [BigchainDB Server Documentation](https://docs.bigchaindb.com/projects/server/en/latest/index.html)
-* [CONTRIBUTING.md](CONTRIBUTING.md) - how to contribute
+* [CONTRIBUTING.md](.github/CONTRIBUTING.md) - how to contribute
 * [Community guidelines](CODE_OF_CONDUCT.md)
 * [Open issues](https://github.com/bigchaindb/bigchaindb/issues)
 * [Open pull requests](https://github.com/bigchaindb/bigchaindb/pulls)

README_cn.md

@@ -0,0 +1,77 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
+<!--- There is no shield to get the latest version
+(including pre-release versions) from PyPI,
+so show the latest GitHub release instead.
+--->
+
+[![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master)
+[![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases)
+[![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/BigchainDB/)
+[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.com/bigchaindb/bigchaindb)
+[![Documentation Status](https://readthedocs.org/projects/bigchaindb-server/badge/?version=latest)](https://docs.bigchaindb.com/projects/server/en/latest/)
+[![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+# BigchainDB 服务器
+
+BigchainDB 是区块链数据库. 这是 _BigchainDB 服务器_ 的仓库.
+
+## 基础知识
+
+* [尝试快速开始](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html)
+* [阅读 BigchainDB 2.0 白皮书](https://www.bigchaindb.com/whitepaper/)
+* [查阅漫游指南](https://www.bigchaindb.com/developers/guide/)
+
+## 运行和测试 `master` 分支的 BigchainDB 服务器
+
+运行和测试最新版本的 BigchainDB 服务器非常简单. 确认你有安装最新版本的 [Docker Compose](https://docs.docker.com/compose/install/). 当你准备好了, 打开一个终端并运行:
+
+```text
+git clone https://github.com/bigchaindb/bigchaindb.git
+cd bigchaindb
+make run
+```
+
+BigchainDB 应该可以通过 `http://localhost:9984/` 访问.
+
+这里也有一些其他的命令你可以运行:
+
+* `make start`: 通过源码和守护进程的方式运行 BigchainDB (通过 `make stop` 停止).
+* `make stop`: 停止运行 BigchainDB.
+* `make logs`: 附在日志上.
+* `make test`: 运行所有单元和验收测试.
+* `make test-unit-watch`: 运行所有测试并等待. 每次更改代码时都会再次运行测试.
+* `make cov`: 检查代码覆盖率并在浏览器中打开结果.
+* `make doc`: 生成 HTML 文档并在浏览器中打开它.
+* `make clean`: 删除所有构建, 测试, 覆盖和 Python 生成物.
+* `make reset`: 停止并移除所有容器. 警告: 您将丢失存储在 BigchainDB 中的所有数据.
+
+查看所有可用命令, 请运行 `make`.
+
+## 一般人员链接
+
+* [BigchainDB.com](https://www.bigchaindb.com/) - BigchainDB 主网站, 包括新闻订阅
+* [路线图](https://github.com/bigchaindb/org/blob/master/ROADMAP.md)
+* [博客](https://medium.com/the-bigchaindb-blog)
+* [推特](https://twitter.com/BigchainDB)
+
+## 开发人员链接
+
+* [所有的 BigchainDB 文档](https://docs.bigchaindb.com/en/latest/)
+* [BigchainDB 服务器 文档](https://docs.bigchaindb.com/projects/server/en/latest/index.html)
+* [CONTRIBUTING.md](.github/CONTRIBUTING.md) - how to contribute
+* [社区指南](CODE_OF_CONDUCT.md)
+* [公开问题](https://github.com/bigchaindb/bigchaindb/issues)
+* [公开的 pull request](https://github.com/bigchaindb/bigchaindb/pulls)
+* [Gitter 聊天室](https://gitter.im/bigchaindb/bigchaindb)
+
+## 法律声明
+
+* [许可](LICENSES.md) - 开源代码 & 开源内容
+* [印记](https://www.bigchaindb.com/imprint/)
+* [联系我们](https://www.bigchaindb.com/contact/)

README_kor.md

@@ -0,0 +1,65 @@
+[![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master)
+[![Latest release](https://img.shields.io/github/release/bigchaindb/bigchaindb/all.svg)](https://github.com/bigchaindb/bigchaindb/releases)
+[![Status on PyPI](https://img.shields.io/pypi/status/bigchaindb.svg)](https://pypi.org/project/BigchainDB/)
+[![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.org/bigchaindb/bigchaindb)
+[![Documentation Status](https://readthedocs.org/projects/bigchaindb-server/badge/?version=latest)](https://docs.bigchaindb.com/projects/server/en/latest/)
+[![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+# BigchainDB 서버
+
+BigchaingDB는 블록체인 데이터베이스입니다. 이 저장소는 _BigchaingDB 서버_를 위한 저장소입니다.
+
+### 기본 사항
+
+* [빠른 시작 사용해보기](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html)
+* [BigchainDB 2.0 백서 읽기](https://www.bigchaindb.com/whitepaper/)
+* [BigchainDB에 대한 _Hitchiker's Guide_를 확인십시오.](https://www.bigchaindb.com/developers/guide/)
+
+### `master` Branch에서 BigchainDB 서버 실행 및 테스트
+
+BigchaingDB 서버의 최신 버전을 실행하고 테스트하는 것은 어렵지 않습니다. [Docker Compose](https://docs.docker.com/compose/install/)의 최신 버전이 설치되어 있는지 확인하십시오. 준비가 되었다면,  터미널에서 다음을 실행하십시오.
+
+```text
+git clone https://github.com/bigchaindb/bigchaindb.git
+cd bigchaindb
+make run
+```
+
+이제 BigchainDB는 `http://localhost:9984/`에 연결되어야 합니다.
+
+또한, 실행시키기 위한  다른 명령어들도 있습니다.
+
+* `make start` : 소스로부터 BigchainDB를 실행하고 데몬화합니다. \(이는 `make stop` 을 하면 중지합니다.\)
+* `make stop` : BigchainDB를 중지합니다.
+* `make logs` : 로그에 첨부합니다.
+* `make text` : 모든 유닛과 허가 테스트를 실행합니다.
+* `make test-unit-watch` : 모든 테스트를 수행하고 기다립니다. 코드를 변경할 때마다 테스트는 다시 실행될 것입니다.
+* `make cov` : 코드 커버리지를 확인하고 브라우저에서 결과를 엽니다.
+* `make doc` : HTML 문서를 만들고, 브라우저에서 엽니다.
+* `make clean` : 모든 빌드와 테스트, 커버리지 및 파이썬 아티팩트를 제거합니다.
+* `make reset` : 모든 컨테이너들을 중지하고 제거합니다. 경고 : BigchainDB에 저장된 모든 데이터를 잃을 수 있습니다.
+
+사용 가능한 모든 명령어를 보기 위해서는 `make` 를 실행하십시오.
+
+### 모두를 위한 링크들
+
+* [BigchainDB.com ](https://www.bigchaindb.com/)- 뉴스 레터 가입을 포함하는 BigchainDB 주요 웹 사이트
+* [로드맵](https://github.com/bigchaindb/org/blob/master/ROADMAP.md)
+* [블로그](https://medium.com/the-bigchaindb-blog)
+* [트위터](https://twitter.com/BigchainDB)
+
+### 개발자들을 위한 링크들
+
+* [모든 BigchainDB 문서](https://docs.bigchaindb.com/en/latest/)
+* [BigchainDB 서버 문서](https://docs.bigchaindb.com/projects/server/en/latest/index.html)
+* [CONTRIBUTING.md](https://github.com/bigchaindb/bigchaindb/blob/master/.github/CONTRIBUTING.md) - 기여를 하는 방법
+* [커뮤니티 가이드라인](https://github.com/bigchaindb/bigchaindb/blob/master/CODE_OF_CONDUCT.md)
+* [이슈 작성](https://github.com/bigchaindb/bigchaindb/issues)
+* [pull request 하기](https://github.com/bigchaindb/bigchaindb/pulls)
+* [Gitter 채팅방](https://gitter.im/bigchaindb/bigchaindb)
+
+### 합법
+
+* [라이선스](https://github.com/bigchaindb/bigchaindb/blob/master/LICENSES.md) - 오픈 소스 & 오픈 콘텐츠
+* [발행](https://www.bigchaindb.com/imprint/)
+* [연락처](https://www.bigchaindb.com/contact/)

RELEASE_PROCESS.md

@@ -1,72 +1,101 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Our Release Process
 
-The release process for BigchainDB server differs slightly depending on whether it's a minor or a patch release.
+## Notes
 
-BigchainDB follows 
+BigchainDB follows
 [the Python form of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme)
 (i.e. MAJOR.MINOR.PATCH),
 which is almost identical
-to [regular semantic versioning](http://semver.org/)
-except release candidates are labelled like
-`3.4.5rc2` not `3.4.5-rc2` (with no hyphen).
+to [regular semantic versioning](http://semver.org/), but there's no hyphen, e.g.
 
+- `0.9.0` for a typical final release
+- `4.5.2a1` not `4.5.2-a1` for the first Alpha release
+- `3.4.5rc2` not `3.4.5-rc2` for Release Candidate 2
 
-## Minor release
+**Note 1:** For Git tags (which are used to identify releases on GitHub), we append a `v` in front. For example, the Git tag for version `2.0.0a1` was `v2.0.0a1`.
 
-A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server.
+**Note 2:** For Docker image tags (e.g. on Docker Hub), we use longer version names for Alpha, Beta and Release Candidate releases. For example, the Docker image tag for version `2.0.0a2` was `2.0.0-alpha2`.
 
-1. Update the `CHANGELOG.md` file in master
-1. In `k8s/bigchaindb/bigchaindb-dep.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.) Commit that change to master
-1. Create and checkout a new branch for the minor release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` (*not* 0.9.0, this new branch will be for e.g. 0.9.0, 0.9.1, 0.9.2, etc. each of which will be identified by a tagged commit)
-1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end)
-1. Commit that change, and push the new branch to GitHub
-1. On GitHub, use the new branch to create a new pull request and wait for all the tests to pass
-1. Follow steps outlined in [Common Steps](#common-steps)
-1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev`. (Exception: If you just released `X.Y.Zrc1` then increment the minor version to `X.Y.Zrc2`.) This step is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available).
-1. Go to [Docker Hub](https://hub.docker.com/), sign in, go to Settings - Build Settings, and under the build with Docker Tag Name equal to `latest`, change the Name to the number of the new release, e.g. `0.9`
+We use `0.9` and `0.9.0` as example version and short-version values below. You should replace those with the correct values for your new version.
 
-Congratulations, you have released BigchainDB!
+We follow [BEP-1](https://github.com/bigchaindb/BEPs/tree/master/1), which is our variant of C4, the Collective Code Construction Contract, so a release is just a [tagged commit](https://git-scm.com/book/en/v2/Git-Basics-Tagging) on the `master` branch, i.e. a label for a particular Git commit.
 
-## Patch release
+The following steps are what we do to release a new version of _BigchainDB Server_. The steps to release the Python Driver are similar but not the same.
 
-A patch release is similar to a minor release, but piggybacks on an existing minor release branch:
+## Steps
 
-1. Check out the minor release branch, e.g. `0.9`
-1. Apply the changes you want, e.g. using `git cherry-pick`.
-1. Update the `CHANGELOG.md` file
-1. Increment the patch version in `bigchaindb/version.py`, e.g. `0.9.1`
-1. Commit that change
-1. In `k8s/bigchaindb/bigchaindb-dep.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.9.0` and change the version number to the new version number, e.g. `0.9.1`. (This is the Docker image that Kubernetes should pull from Docker Hub.)
-1. Commit that change
-1. Push the updated minor release branch to GitHub
-1. Follow steps outlined in [Common Steps](#common-steps)
-1. Cherry-pick the `CHANGELOG.md` update commit (made above) to the `master` branch
+1. Create a pull request where you make the following changes:
 
-## Common steps
+   - Update `CHANGELOG.md`
+   - Update all Docker image tags in all Kubernetes YAML files (in the `k8s/` directory).
+     For example, in the files:
 
-These steps are common between minor and patch releases:
+     - `k8s/bigchaindb/bigchaindb-ss.yaml` and
+     - `k8s/dev-setup/bigchaindb.yaml`
 
-1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases)
-   and click the "Draft a new release" button
-1. Fill in the details:
-   - Tag version: version number preceeded by 'v', e.g. "v0.9.1"
-   - Target: the release branch that was just pushed
-   - Title: Same as tag name above, e.g "v0.9.1"
-   - Description: The body of the changelog entry (Added, Changed, etc.)
-1. Click "Publish release" to publish the release on GitHub
-1. Make sure your local Git is in the same state as the release: e.g. `git fetch <remote-name>` and `git checkout v0.9.1`
-1. Make sure you have a `~/.pypirc` file containing credentials for PyPI
-1. Do a `make release` to build and publish the new `bigchaindb` package on PyPI
-1. [Login to readthedocs.org](https://readthedocs.org/accounts/login/)
-   as a maintainer of the BigchainDB Server docs, and:
+     find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.)
+     Keep in mind that this is a _Docker image tag_ so our naming convention is
+     a bit different; see Note 2 in the **Notes** section above.
+   - In `bigchaindb/version.py`:
+     - update `__version__` to e.g. `0.9.0` (with no `.dev` on the end)
+     - update `__short_version__` to e.g. `0.9` (with no `.dev` on the end)
+   - In the docs about installing BigchainDB (and Tendermint), and in the associated scripts, recommend/install a version of Tendermint that _actually works_ with the soon-to-be-released version of BigchainDB. You can find all such references by doing a search for the previously-recommended version number, such as `0.31.5`.
+   - In `setup.py`, _maybe_ update the development status item in the `classifiers` list. For example, one allowed value is `"Development Status :: 5 - Production/Stable"`. The [allowed values are listed at pypi.python.org](https://pypi.python.org/pypi?%3Aaction=list_classifiers).
+
+2. **Wait for all the tests to pass!**
+3. Merge the pull request into the `master` branch.
+4. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases)
+   and click the "Draft a new release" button.
+5. Fill in the details:
+   - **Tag version:** version number preceded by `v`, e.g. `v0.9.1`
+   - **Target:** the last commit that was just merged. In other words, that commit will get a Git tag with the value given for tag version above.
+   - **Title:** Same as tag version above, e.g `v0.9.1`
+   - **Description:** The body of the changelog entry (Added, Changed, etc.)
+6. Click "Publish release" to publish the release on GitHub.
+7. On your local computer, make sure you're on the `master` branch and that it's up-to-date with the `master` branch in the bigchaindb/bigchaindb repository (e.g. `git pull upstream master`). We're going to use that to push a new `bigchaindb` package to PyPI.
+8. Make sure you have a `~/.pypirc` file containing credentials for PyPI.
+9. Do `make release` to build and publish the new `bigchaindb` package on PyPI. For this step you need to have `twine` installed. If you get an error like `Makefile:135: recipe for target 'clean-pyc' failed` then try doing
+   ```text
+   sudo chown -R $(whoami):$(whoami) .
+   ```
+10. [Log in to readthedocs.org](https://readthedocs.org/accounts/login/) and go to the **BigchainDB Server** project, then:
+   - Click on "Builds", select "latest" from the drop-down menu, then click the "Build Version:" button.
+   - Wait for the build of "latest" to finish. This can take a few minutes.
    - Go to Admin --> Advanced Settings
      and make sure that "Default branch:" (i.e. what "latest" points to)
      is set to the new release's tag, e.g. `v0.9.1`.
-     (Don't miss the 'v' in front.)
+     (It won't be an option if you didn't wait for the build of "latest" to finish.)
+     Then scroll to the bottom and click "Save".
    - Go to Admin --> Versions
      and under **Choose Active Versions**, do these things:
      1. Make sure that the new version's tag is "Active" and "Public"
-     2. Make sure the new version's branch
-        (without the 'v' in front) is _not_ active.
-     3. Make sure the **stable** branch is _not_ active.
-     4. Scroll to the bottom of the page and click the Submit button.
+     2. Make sure the **stable** branch is _not_ active.
+     3. Scroll to the bottom of the page and click "Save".
+11. Go to [Docker Hub](https://hub.docker.com/) and sign in, then:
+   - Click on "Organizations"
+   - Click on "bigchaindb"
+   - Click on "bigchaindb/bigchaindb"
+   - Click on "Build Settings"
+   - Find the row where "Docker Tag Name" equals `latest`
+     and change the value of "Name" to the name (Git tag)
+     of the new release, e.g. `v0.9.0`.
+   - If the release is an Alpha, Beta or Release Candidate release,
+     then a new row must be added.
+     You can do that by clicking the green "+" (plus) icon.
+     The contents of the new row should be similar to the existing rows
+     of previous releases like that.
+   - Click on "Tags"
+   - Delete the "latest" tag (so we can rebuild it)
+   - Click on "Build Settings" again
+   - Click on the "Trigger" button for the "latest" tag and make sure it worked by clicking on "Tags" again
+   - If the release is an Alpha, Beta or Release Candidate release,
+     then click on the "Trigger" button for that tag as well.
+
+Congratulations, you have released a new version of BigchainDB Server!

ROADMAP.md

@@ -1,3 +1,10 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # BigchainDB Roadmap
 
 We moved the BigchainDB Roadmap to the bigchaindb/org repository; see:

acceptance/README.md

@@ -0,0 +1,27 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
+# Acceptance test suite
+This directory contains the acceptance test suite for BigchainDB.
+
+The suite uses Docker Compose to set up a single BigchainDB node, run all tests, and finally stop the node. In the future we will add support for a four node network setup.
+
+## Running the tests
+It should be as easy as `make test-acceptance`.
+
+Note that `make test-acceptance` will take some time to start the node and shutting it down. If you are developing a test, or you wish to run a specific test in the acceptance test suite, first start the node with `make start`. After the node is running, you can run `pytest` inside the `python-acceptance` container with:
+
+```bash
+docker-compose run --rm python-acceptance pytest <use whatever option you need>
+```
+
+## Writing and documenting the tests
+Tests are sometimes difficult to read. For acceptance tests, we try to be really explicit on what the test is doing, so please write code that is *simple* and easy to understand. We decided to use literate-programming documentation. To generate the documentation run:
+
+```bash
+make doc-acceptance
+```

acceptance/python/.gitignore

@@ -0,0 +1 @@
+docs

acceptance/python/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.6.3
+
+RUN mkdir -p /src
+RUN pip install --upgrade \
+	pycco \
+	websocket-client~=0.47.0 \
+	pytest~=3.0 \
+	bigchaindb-driver~=0.6.2 \
+	blns

acceptance/python/src/test_basic.py

@@ -0,0 +1,125 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# # Basic Acceptance Test
+# Here we check that the primitives of the system behave as expected.
+# As you will see, this script tests basic stuff like:
+#
+# - create a transaction
+# - check if the transaction is stored
+# - check for the outputs of a given public key
+# - transfer the transaction to another key
+#
+# We run a series of checks for each steps, that is retrieving the transaction from
+# the remote system, and also checking the `outputs` of a given public key.
+#
+# This acceptance test is a rip-off of our
+# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html).
+
+# ## Imports
+# We need some utils from the `os` package, we will interact with
+# env variables.
+import os
+
+# For this test case we import and use the Python Driver.
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_basic():
+    # ## Set up a connection to BigchainDB
+    # To use BighainDB we need a connection. Here we create one. By default we
+    # connect to localhost, but you can override this value using the env variable
+    # called `BIGCHAINDB_ENDPOINT`, a valid value must include the schema:
+    # `https://example.com:9984`
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+
+    # ## Create keypairs
+    # This test requires the interaction between two actors with their own keypair.
+    # The two keypairs will be called—drum roll—Alice and Bob.
+    alice, bob = generate_keypair(), generate_keypair()
+
+    # ## Alice registers her bike in BigchainDB
+    # Alice has a nice bike, and here she creates the "digital twin"
+    # of her bike.
+    bike = {'data': {'bicycle': {'serial_number': 420420}}}
+
+    # She prepares a `CREATE` transaction...
+    prepared_creation_tx = bdb.transactions.prepare(
+            operation='CREATE',
+            signers=alice.public_key,
+            asset=bike)
+
+    # ... and she fulfills it with her private key.
+    fulfilled_creation_tx = bdb.transactions.fulfill(
+            prepared_creation_tx,
+            private_keys=alice.private_key)
+
+    # We will use the `id` of this transaction several time, so we store it in
+    # a variable with a short and easy name
+    bike_id = fulfilled_creation_tx['id']
+
+    # Now she is ready to send it to the BigchainDB Network.
+    sent_transfer_tx = bdb.transactions.send_commit(fulfilled_creation_tx)
+
+    # And just to be 100% sure, she also checks if she can retrieve
+    # it from the BigchainDB node.
+    assert bdb.transactions.retrieve(bike_id), 'Cannot find transaction {}'.format(bike_id)
+
+    # Alice is now the proud owner of one unspent asset.
+    assert len(bdb.outputs.get(alice.public_key, spent=False)) == 1
+    assert bdb.outputs.get(alice.public_key)[0]['transaction_id'] == bike_id
+
+    # ## Alice transfers her bike to Bob
+    # After registering her bike, Alice is ready to transfer it to Bob.
+    # She needs to create a new `TRANSFER` transaction.
+
+    # A `TRANSFER` transaction contains a pointer to the original asset. The original asset
+    # is identified by the `id` of the `CREATE` transaction that defined it.
+    transfer_asset = {'id': bike_id}
+
+    # Alice wants to spend the one and only output available, the one with index `0`.
+    output_index = 0
+    output = fulfilled_creation_tx['outputs'][output_index]
+
+    # Here, she defines the `input` of the `TRANSFER` transaction. The `input` contains
+    # several keys:
+    #
+    # - `fulfillment`, taken from the previous `CREATE` transaction.
+    # - `fulfills`, that specifies which condition she is fulfilling.
+    # - `owners_before`.
+    transfer_input = {'fulfillment': output['condition']['details'],
+                      'fulfills': {'output_index': output_index,
+                                   'transaction_id': fulfilled_creation_tx['id']},
+                      'owners_before': output['public_keys']}
+
+    # Now that all the elements are set, she creates the actual transaction...
+    prepared_transfer_tx = bdb.transactions.prepare(
+            operation='TRANSFER',
+            asset=transfer_asset,
+            inputs=transfer_input,
+            recipients=bob.public_key)
+
+    # ... and signs it with her private key.
+    fulfilled_transfer_tx = bdb.transactions.fulfill(
+            prepared_transfer_tx,
+            private_keys=alice.private_key)
+
+    # She finally sends the transaction to a BigchainDB node.
+    sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx)
+
+    # And just to be 100% sure, she also checks if she can retrieve
+    # it from the BigchainDB node.
+    assert bdb.transactions.retrieve(fulfilled_transfer_tx['id']) == sent_transfer_tx
+
+    # Now Alice has zero unspent transactions.
+    assert len(bdb.outputs.get(alice.public_key, spent=False)) == 0
+
+    # While Bob has one.
+    assert len(bdb.outputs.get(bob.public_key, spent=False)) == 1
+
+    # Bob double checks what he got was the actual bike.
+    bob_tx_id = bdb.outputs.get(bob.public_key, spent=False)[0]['transaction_id']
+    assert bdb.transactions.retrieve(bob_tx_id) == sent_transfer_tx

acceptance/python/src/test_divisible_asset.py

@@ -0,0 +1,181 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# # Divisible assets integration testing
+# This test checks if we can successfully divide assets.
+# The script tests various things like:
+#
+# - create a transaction with a divisible asset and issue them to someone
+# - check if the transaction is stored and has the right amount of tokens
+# - spend some tokens
+# - try to spend more tokens than available
+#
+# We run a series of checks for each step, that is retrieving
+# the transaction from the remote system, and also checking the `amount`
+# of a given transaction.
+#
+# This integration test is a rip-off of our
+# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html).
+
+# ## Imports
+# We need some utils from the `os` package, we will interact with
+# env variables.
+# We need the `pytest` package to catch the `BadRequest` exception properly.
+# And of course, we also need the `BadRequest`.
+import os
+import pytest
+from bigchaindb_driver.exceptions import BadRequest
+
+# For this test case we import and use the Python Driver.
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_divisible_assets():
+    # ## Set up a connection to BigchainDB
+    # Check [test_basic.py](./test_basic.html) to get some more details
+    # about the endpoint.
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+
+    # Oh look, it is Alice again and she brought her friend Bob along.
+    alice, bob = generate_keypair(), generate_keypair()
+
+    # ## Alice creates a time sharing token
+    # Alice wants to go on vacation, while Bobs bike just broke down.
+    # Alice decides to rent her bike to Bob while she is gone.
+    # So she prepares a `CREATE` transaction to issues 10 tokens.
+    # First, she prepares an asset for a time sharing token. As you can see in
+    # the description, Bob and Alice agree that each token can be used to ride
+    # the bike for one hour.
+
+    bike_token = {
+        'data': {
+            'token_for': {
+                'bike': {
+                    'serial_number': 420420
+                }
+            },
+            'description': 'Time share token. Each token equals one hour of riding.',
+        },
+    }
+
+    # She prepares a `CREATE` transaction and issues 10 tokens.
+    # Here, Alice defines in a tuple that she wants to assign
+    # these 10 tokens to Bob.
+    prepared_token_tx = bdb.transactions.prepare(
+        operation='CREATE',
+        signers=alice.public_key,
+        recipients=[([bob.public_key], 10)],
+        asset=bike_token)
+
+    # She fulfills and sends the transaction.
+    fulfilled_token_tx = bdb.transactions.fulfill(
+        prepared_token_tx,
+        private_keys=alice.private_key)
+
+    bdb.transactions.send_commit(fulfilled_token_tx)
+
+    # We store the `id` of the transaction to use it later on.
+    bike_token_id = fulfilled_token_tx['id']
+
+    # Let's check if the transaction was successful.
+    assert bdb.transactions.retrieve(bike_token_id), \
+        'Cannot find transaction {}'.format(bike_token_id)
+
+    # Bob owns 10 tokens now.
+    assert bdb.transactions.retrieve(bike_token_id)['outputs'][0][
+        'amount'] == '10'
+
+    # ## Bob wants to use the bike
+    # Now that Bob got the tokens and the sun is shining, he wants to get out
+    # with the bike for three hours.
+    # To use the bike he has to send the tokens back to Alice.
+    # To learn about the details of transferring a transaction check out
+    # [test_basic.py](./test_basic.html)
+    transfer_asset = {'id': bike_token_id}
+
+    output_index = 0
+    output = fulfilled_token_tx['outputs'][output_index]
+    transfer_input = {'fulfillment': output['condition']['details'],
+                      'fulfills': {'output_index': output_index,
+                                   'transaction_id': fulfilled_token_tx[
+                                       'id']},
+                      'owners_before': output['public_keys']}
+
+    # To use the tokens Bob has to reassign 7 tokens to himself and the
+    # amount he wants to use to Alice.
+    prepared_transfer_tx = bdb.transactions.prepare(
+        operation='TRANSFER',
+        asset=transfer_asset,
+        inputs=transfer_input,
+        recipients=[([alice.public_key], 3), ([bob.public_key], 7)])
+
+    # He signs and sends the transaction.
+    fulfilled_transfer_tx = bdb.transactions.fulfill(
+        prepared_transfer_tx,
+        private_keys=bob.private_key)
+
+    sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx)
+
+    # First, Bob checks if the transaction was successful.
+    assert bdb.transactions.retrieve(
+        fulfilled_transfer_tx['id']) == sent_transfer_tx
+    # There are two outputs in the transaction now.
+    # The first output shows that Alice got back 3 tokens...
+    assert bdb.transactions.retrieve(
+        fulfilled_transfer_tx['id'])['outputs'][0]['amount'] == '3'
+
+    # ... while Bob still has 7 left.
+    assert bdb.transactions.retrieve(
+        fulfilled_transfer_tx['id'])['outputs'][1]['amount'] == '7'
+
+    # ## Bob wants to ride the bike again
+    # It's been a week and Bob wants to right the bike again.
+    # Now he wants to ride for 8 hours, that's a lot Bob!
+    # He prepares the transaction again.
+
+    transfer_asset = {'id': bike_token_id}
+    # This time we need an `output_index` of 1, since we have two outputs
+    # in the `fulfilled_transfer_tx` we created before. The first output with
+    # index 0 is for Alice and the second output is for Bob.
+    # Since Bob wants to spend more of his tokens he has to provide the
+    # correct output with the correct amount of tokens.
+    output_index = 1
+
+    output = fulfilled_transfer_tx['outputs'][output_index]
+
+    transfer_input = {'fulfillment': output['condition']['details'],
+                      'fulfills': {'output_index': output_index,
+                                   'transaction_id': fulfilled_transfer_tx['id']},
+                      'owners_before': output['public_keys']}
+
+    # This time Bob only provides Alice in the `recipients` because he wants
+    # to spend all his tokens
+    prepared_transfer_tx = bdb.transactions.prepare(
+        operation='TRANSFER',
+        asset=transfer_asset,
+        inputs=transfer_input,
+        recipients=[([alice.public_key], 8)])
+
+    fulfilled_transfer_tx = bdb.transactions.fulfill(
+        prepared_transfer_tx,
+        private_keys=bob.private_key)
+
+    # Oh Bob, what have you done?! You tried to spend more tokens than you had.
+    # Remember Bob, last time you spent 3 tokens already,
+    # so you only have 7 left.
+    with pytest.raises(BadRequest) as error:
+        bdb.transactions.send_commit(fulfilled_transfer_tx)
+
+    # Now Bob gets an error saying that the amount he wanted to spent is
+    # higher than the amount of tokens he has left.
+    assert error.value.args[0] == 400
+    message = 'Invalid transaction (AmountError): The amount used in the ' \
+              'inputs `7` needs to be same as the amount used in the ' \
+              'outputs `8`'
+    assert error.value.args[2]['message'] == message
+
+    # We have to stop this test now, I am sorry, but Bob is pretty upset
+    # about his mistake. See you next time :)

acceptance/python/src/test_double_spend.py

@@ -0,0 +1,48 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# # Double Spend testing
+# This test challenge the system with double spends.
+
+import os
+from uuid import uuid4
+from threading import Thread
+import queue
+
+import bigchaindb_driver.exceptions
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_double_create():
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+    alice = generate_keypair()
+
+    results = queue.Queue()
+
+    tx = bdb.transactions.fulfill(
+            bdb.transactions.prepare(
+                operation='CREATE',
+                signers=alice.public_key,
+                asset={'data': {'uuid': str(uuid4())}}),
+            private_keys=alice.private_key)
+
+    def send_and_queue(tx):
+        try:
+            bdb.transactions.send_commit(tx)
+            results.put('OK')
+        except bigchaindb_driver.exceptions.TransportError as e:
+            results.put('FAIL')
+
+    t1 = Thread(target=send_and_queue, args=(tx, ))
+    t2 = Thread(target=send_and_queue, args=(tx, ))
+
+    t1.start()
+    t2.start()
+
+    results = [results.get(timeout=2), results.get(timeout=2)]
+
+    assert results.count('OK') == 1
+    assert results.count('FAIL') == 1

acceptance/python/src/test_multiple_owners.py

@@ -0,0 +1,126 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# # Multiple owners integration testing
+# This test checks if we can successfully create and transfer a transaction
+# with multiple owners.
+# The script tests various things like:
+#
+# - create a transaction with multiple owners
+# - check if the transaction is stored and has the right amount of public keys
+# - transfer the transaction to a third person
+#
+# We run a series of checks for each step, that is retrieving
+# the transaction from the remote system, and also checking the public keys
+# of a given transaction.
+#
+# This integration test is a rip-off of our
+# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html).
+
+# ## Imports
+# We need some utils from the `os` package, we will interact with
+# env variables.
+import os
+
+# For this test case we import and use the Python Driver.
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_multiple_owners():
+    # ## Set up a connection to BigchainDB
+    # Check [test_basic.py](./test_basic.html) to get some more details
+    # about the endpoint.
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+
+    # Hey Alice and Bob, nice to see you again!
+    alice, bob = generate_keypair(), generate_keypair()
+
+    # ## Alice and Bob create a transaction
+    # Alice and Bob just moved into a shared flat, no one can afford these
+    # high rents anymore. Bob suggests to get a dish washer for the
+    # kitchen. Alice agrees and here they go, creating the asset for their
+    # dish washer.
+    dw_asset = {
+        'data': {
+            'dish washer': {
+                'serial_number': 1337
+            }
+        }
+    }
+
+    # They prepare a `CREATE` transaction. To have multiple owners, both
+    # Bob and Alice need to be the recipients.
+    prepared_dw_tx = bdb.transactions.prepare(
+        operation='CREATE',
+        signers=alice.public_key,
+        recipients=(alice.public_key, bob.public_key),
+        asset=dw_asset)
+
+    # Now they both sign the transaction by providing their private keys.
+    # And send it afterwards.
+    fulfilled_dw_tx = bdb.transactions.fulfill(
+        prepared_dw_tx,
+        private_keys=[alice.private_key, bob.private_key])
+
+    bdb.transactions.send_commit(fulfilled_dw_tx)
+
+    # We store the `id` of the transaction to use it later on.
+    dw_id = fulfilled_dw_tx['id']
+
+    # Let's check if the transaction was successful.
+    assert bdb.transactions.retrieve(dw_id), \
+        'Cannot find transaction {}'.format(dw_id)
+
+    # The transaction should have two public keys in the outputs.
+    assert len(
+        bdb.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2
+
+    # ## Alice and Bob transfer a transaction to Carol.
+    # Alice and Bob save a lot of money living together. They often go out
+    # for dinner and don't cook at home. But now they don't have any dishes to
+    # wash, so they decide to sell the dish washer to their friend Carol.
+
+    # Hey Carol, nice to meet you!
+    carol = generate_keypair()
+
+    # Alice and Bob prepare the transaction to transfer the dish washer to
+    # Carol.
+    transfer_asset = {'id': dw_id}
+
+    output_index = 0
+    output = fulfilled_dw_tx['outputs'][output_index]
+    transfer_input = {'fulfillment': output['condition']['details'],
+                      'fulfills': {'output_index': output_index,
+                                   'transaction_id': fulfilled_dw_tx[
+                                       'id']},
+                      'owners_before': output['public_keys']}
+
+    # Now they create the transaction...
+    prepared_transfer_tx = bdb.transactions.prepare(
+        operation='TRANSFER',
+        asset=transfer_asset,
+        inputs=transfer_input,
+        recipients=carol.public_key)
+
+    # ... and sign it with their private keys, then send it.
+    fulfilled_transfer_tx = bdb.transactions.fulfill(
+        prepared_transfer_tx,
+        private_keys=[alice.private_key, bob.private_key])
+
+    sent_transfer_tx = bdb.transactions.send_commit(fulfilled_transfer_tx)
+
+    # They check if the transaction was successful.
+    assert bdb.transactions.retrieve(
+        fulfilled_transfer_tx['id']) == sent_transfer_tx
+
+    # The owners before should include both Alice and Bob.
+    assert len(
+        bdb.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][
+            'owners_before']) == 2
+
+    # While the new owner is Carol.
+    assert bdb.transactions.retrieve(fulfilled_transfer_tx['id'])[
+           'outputs'][0]['public_keys'][0] == carol.public_key

acceptance/python/src/test_naughty_strings.py

@@ -0,0 +1,101 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# ## Testing potentially hazardous strings
+# This test uses a library of `naughty` strings (code injections, weird unicode chars., etc.) as both keys and values.
+# We look for either a successful tx, or in the case that we use a naughty string as a key, and it violates some key
+# constraints, we expect to receive a well formatted error message.
+
+# ## Imports
+# We need some utils from the `os` package, we will interact with
+# env variables.
+import os
+
+# Since the naughty strings get encoded and decoded in odd ways,
+# we'll use a regex to sweep those details under the rug.
+import re
+
+# We'll use a nice library of naughty strings...
+from blns import blns
+
+# And parameterize our test so each one is treated as a separate test case
+import pytest
+
+# For this test case we import and use the Python Driver.
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+from bigchaindb_driver.exceptions import BadRequest
+
+naughty_strings = blns.all()
+
+
+# This is our base test case, but we'll reuse it to send naughty strings as both keys and values.
+def send_naughty_tx(asset, metadata):
+    # ## Set up a connection to BigchainDB
+    # Check [test_basic.py](./test_basic.html) to get some more details
+    # about the endpoint.
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+
+    # Here's Alice.
+    alice = generate_keypair()
+
+    # Alice is in a naughty mood today, so she creates a tx with some naughty strings
+    prepared_transaction = bdb.transactions.prepare(
+        operation='CREATE',
+        signers=alice.public_key,
+        asset=asset,
+        metadata=metadata)
+
+    # She fulfills the transaction
+    fulfilled_transaction = bdb.transactions.fulfill(
+        prepared_transaction,
+        private_keys=alice.private_key)
+
+    # The fulfilled tx gets sent to the BDB network
+    try:
+        sent_transaction = bdb.transactions.send_commit(fulfilled_transaction)
+    except BadRequest as e:
+        sent_transaction = e
+
+    # If her key contained a '.', began with a '$', or contained a NUL character
+    regex = '.*\..*|\$.*|.*\x00.*'
+    key = next(iter(metadata))
+    if re.match(regex, key):
+        # Then she expects a nicely formatted error code
+        status_code = sent_transaction.status_code
+        error = sent_transaction.error
+        regex = (
+            r'\{\s*\n*'
+            r'\s*"message":\s*"Invalid transaction \(ValidationError\):\s*'
+            r'Invalid key name.*The key name cannot contain characters.*\n*'
+            r'\s*"status":\s*400\n*'
+            r'\s*\}\n*')
+        assert status_code == 400
+        assert re.fullmatch(regex, error), sent_transaction
+    # Otherwise, she expects to see her transaction in the database
+    elif 'id' in sent_transaction.keys():
+        tx_id = sent_transaction['id']
+        assert bdb.transactions.retrieve(tx_id)
+    # If neither condition was true, then something weird happened...
+    else:
+        raise TypeError(sent_transaction)
+
+
+@pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings)
+def test_naughty_keys(naughty_string):
+
+    asset = {'data': {naughty_string: 'nice_value'}}
+    metadata = {naughty_string: 'nice_value'}
+
+    send_naughty_tx(asset, metadata)
+
+
+@pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings)
+def test_naughty_values(naughty_string):
+
+    asset = {'data': {'nice_key': naughty_string}}
+    metadata = {'nice_key': naughty_string}
+
+    send_naughty_tx(asset, metadata)

acceptance/python/src/test_stream.py

@@ -0,0 +1,132 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# # Stream Acceptance Test
+# This test checks if the event stream works correctly. The basic idea of this
+# test is to generate some random **valid** transaction, send them to a
+# BigchainDB node, and expect those transactions to be returned by the valid
+# transactions Stream API. During this test, two threads work together,
+# sharing a queue to exchange events.
+#
+# - The *main thread* first creates and sends the transactions to BigchainDB;
+# then it run through all events in the shared queue to check if all
+# transactions sent have been validated by BigchainDB.
+# - The *listen thread* listens to the events coming from BigchainDB and puts
+# them in a queue shared with the main thread.
+
+import os
+import queue
+import json
+from threading import Thread, Event
+from uuid import uuid4
+
+# For this script, we need to set up a websocket connection, that's the reason
+# we import the
+# [websocket](https://github.com/websocket-client/websocket-client) module
+from websocket import create_connection
+
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_stream():
+    # ## Set up the test
+    # We use the env variable `BICHAINDB_ENDPOINT` to know where to connect.
+    # Check [test_basic.py](./test_basic.html) for more information.
+    BDB_ENDPOINT = os.environ.get('BIGCHAINDB_ENDPOINT')
+
+    # *That's pretty bad, but let's do like this for now.*
+    WS_ENDPOINT = 'ws://{}:9985/api/v1/streams/valid_transactions'.format(BDB_ENDPOINT.rsplit(':')[0])
+
+    bdb = BigchainDB(BDB_ENDPOINT)
+
+    # Hello to Alice again, she is pretty active in those tests, good job
+    # Alice!
+    alice = generate_keypair()
+
+    # We need few variables to keep the state, specifically we need `sent` to
+    # keep track of all transactions Alice sent to BigchainDB, while `received`
+    # are the transactions BigchainDB validated and sent back to her.
+    sent = []
+    received = queue.Queue()
+
+    # In this test we use a websocket. The websocket must be started **before**
+    # sending transactions to BigchainDB, otherwise we might lose some
+    # transactions. The `ws_ready` event is used to synchronize the main thread
+    # with the listen thread.
+    ws_ready = Event()
+
+    # ## Listening to events
+    # This is the function run by the complementary thread.
+    def listen():
+        # First we connect to the remote endpoint using the WebSocket protocol.
+        ws = create_connection(WS_ENDPOINT)
+
+        # After the connection has been set up, we can signal the main thread
+        # to proceed (continue reading, it should make sense in a second.)
+        ws_ready.set()
+
+        # It's time to consume all events coming from the BigchainDB stream API.
+        # Every time a new event is received, it is put in the queue shared
+        # with the main thread.
+        while True:
+            result = ws.recv()
+            received.put(result)
+
+    # Put `listen` in a thread, and start it. Note that `listen` is a local
+    # function and it can access all variables in the enclosing function.
+    t = Thread(target=listen, daemon=True)
+    t.start()
+
+    # ## Pushing the transactions to BigchainDB
+    # After starting the listen thread, we wait for it to connect, and then we
+    # proceed.
+    ws_ready.wait()
+
+    # Here we prepare, sign, and send ten different `CREATE` transactions. To
+    # make sure each transaction is different from the other, we generate a
+    # random `uuid`.
+    for _ in range(10):
+        tx = bdb.transactions.fulfill(
+                bdb.transactions.prepare(
+                    operation='CREATE',
+                    signers=alice.public_key,
+                    asset={'data': {'uuid': str(uuid4())}}),
+                private_keys=alice.private_key)
+        # We don't want to wait for each transaction to be in a block. By using
+        # `async` mode, we make sure that the driver returns as soon as the
+        # transaction is pushed to the BigchainDB API. Remember: we expect all
+        # transactions to be in the shared queue: this is a two phase test,
+        # first we send a bunch of transactions, then we check if they are
+        # valid (and, in this case, they should).
+        bdb.transactions.send_async(tx)
+
+        # The `id` of every sent transaction is then stored in a list.
+        sent.append(tx['id'])
+
+    # ## Check the valid transactions coming from BigchainDB
+    # Now we are ready to check if BigchainDB did its job. A simple way to
+    # check if all sent transactions have been processed is to **remove** from
+    # `sent` the transactions we get from the *listen thread*. At one point in
+    # time, `sent` should be empty, and we exit the test.
+    while sent:
+        # To avoid waiting forever, we have an arbitrary timeout of 5
+        # seconds: it should be enough time for BigchainDB to create
+        # blocks, in fact a new block is created every second. If we hit
+        # the timeout, then game over ¯\\\_(ツ)\_/¯
+        try:
+            event = received.get(timeout=5)
+            txid = json.loads(event)['transaction_id']
+        except queue.Empty:
+            assert False, 'Did not receive all expected transactions'
+
+        # Last thing is to try to remove the `txid` from the set of sent
+        # transactions. If this test is running in parallel with others, we
+        # might get a transaction id of another test, and `remove` can fail.
+        # It's OK if this happens.
+        try:
+            sent.remove(txid)
+        except ValueError:
+            pass

benchmark.yml

@@ -1,37 +0,0 @@
-version: '2'
-
-services:
-  bdb:
-    build:
-      context: .
-      dockerfile: Dockerfile-dev
-      args:
-        backend: mongodb
-    volumes:
-      - ./bigchaindb:/usr/src/app/bigchaindb
-      - ./tests:/usr/src/app/tests
-      - ./docs:/usr/src/app/docs
-      - ./k8s:/usr/src/app/k8s
-      - ./setup.py:/usr/src/app/setup.py
-      - ./setup.cfg:/usr/src/app/setup.cfg
-      - ./pytest.ini:/usr/src/app/pytest.ini
-      - ./tox.ini:/usr/src/app/tox.ini
-      - ./scripts:/usr/src/app/scripts
-    environment:
-      BIGCHAINDB_DATABASE_BACKEND: mongodb
-      BIGCHAINDB_DATABASE_HOST: mdb
-      BIGCHAINDB_DATABASE_PORT: 27017
-      BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984
-      BIGCHAINDB_GRAPHITE_HOST: graphite
-    ports:
-      - "9984"
-    command: bigchaindb start
-
-  graphite:
-    image: hopsoft/graphite-statsd
-    ports:
-      - "2003-2004"
-      - "2023-2024"
-      - "8125/udp"
-      - "8126"
-      - "80"

bigchaindb/README.md

@@ -1,22 +1,27 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Overview
 
 A high-level description of the files and subdirectories of BigchainDB.
 
-There are three database tables which underpin BigchainDB: `backlog`, where incoming transactions are held temporarily until they can be consumed; `bigchain`, where blocks of transactions are written permanently; and `votes`, where votes are written permanently.  It is the votes in the `votes` table which must be queried to determine block validity and order. For more in-depth explanation, see [the whitepaper](https://www.bigchaindb.com/whitepaper/).
-
 ## Files
 
-### [`core.py`](./core.py)
+### [`lib.py`](lib.py)
 
-The `Bigchain` class is defined here.  Most operations outlined in the [whitepaper](https://www.bigchaindb.com/whitepaper/) as well as database interactions are found in this file.  This is the place to start if you are interested in implementing a server API, since many of these class methods concern BigchainDB interacting with the outside world.
+The `BigchainDB` class is defined here.  Most node-level operations and database interactions are found in this file.  This is the place to start if you are interested in implementing a server API, since many of these class methods concern BigchainDB interacting with the outside world.
 
 ### [`models.py`](./models.py)
 
 `Block`, `Transaction`, and `Asset` classes are defined here.  The classes mirror the block and transaction structure from the [documentation](https://docs.bigchaindb.com/projects/server/en/latest/data-models/index.html), but also include methods for validation and signing.
 
-### [`consensus.py`](./consensus.py)
+### [`validation.py`](./validation.py)
 
-Base class for consensus methods (verification of votes, blocks, and transactions).  The actual logic is mostly found in `transaction` and `block` models, defined in [`models.py`](./models.py).
+Base class for validation methods (verification of votes, blocks, and transactions).  The actual logic is mostly found in `transaction` and `block` models, defined in [`models.py`](./models.py).
 
 ### [`processes.py`](./processes.py)
 
@@ -28,10 +33,6 @@ Methods for managing the configuration, including loading configuration files, a
 
 ## Folders
 
-### [`pipelines`](./pipelines)
-
-Structure and implementation of various subprocesses started in [`processes.py`](./processes.py).
-
 ### [`commands`](./commands)
 
 Contains code for the [CLI](https://docs.bigchaindb.com/projects/server/en/latest/server-reference/bigchaindb-cli.html) for BigchainDB.

bigchaindb/__init__.py

@@ -1,87 +1,80 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 import copy
 import logging
-import os
 
-from bigchaindb.log.configs import SUBSCRIBER_LOGGING_CONFIG as log_config
+from bigchaindb.log import DEFAULT_LOGGING_CONFIG as log_config
+from bigchaindb.lib import BigchainDB  # noqa
+from bigchaindb.migrations.chain_migration_election import ChainMigrationElection
+from bigchaindb.version import __version__  # noqa
+from bigchaindb.core import App  # noqa
 
 # from functools import reduce
 # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16
 # basically, the port number is 9984
 
-
-_base_database_rethinkdb = {
-    'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'),
-    'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)),
-    'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'),
-}
-
 # The following variable is used by `bigchaindb configure` to
 # prompt the user for database values. We cannot rely on
-# _base_database_rethinkdb.keys() or _base_database_mongodb.keys()
-# because dicts are unordered. I tried to configure
+# _base_database_localmongodb.keys() because dicts are unordered.
+# I tried to configure
 
 _database_keys_map = {
-    'mongodb': ('host', 'port', 'name', 'replicaset'),
-    'rethinkdb': ('host', 'port', 'name')
+    'localmongodb': ('host', 'port', 'name'),
 }
 
-_base_database_mongodb = {
-    'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'),
-    'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)),
-    'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'),
-    'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'),
-    'login': os.environ.get('BIGCHAINDB_DATABASE_LOGIN'),
-    'password': os.environ.get('BIGCHAINDB_DATABASE_PASSWORD')
+_base_database_localmongodb = {
+    'host': 'localhost',
+    'port': 27017,
+    'name': 'bigchain',
+    'replicaset': None,
+    'login': None,
+    'password': None,
 }
 
-_database_rethinkdb = {
-    'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'),
+_database_localmongodb = {
+    'backend': 'localmongodb',
     'connection_timeout': 5000,
     'max_tries': 3,
+    'ssl': False,
+    'ca_cert': None,
+    'certfile': None,
+    'keyfile': None,
+    'keyfile_passphrase': None,
+    'crlfile': None,
 }
-_database_rethinkdb.update(_base_database_rethinkdb)
-
-_database_mongodb = {
-    'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'),
-    'connection_timeout': 5000,
-    'max_tries': 3,
-    'ssl': bool(os.environ.get('BIGCHAINDB_DATABASE_SSL', False)),
-    'ca_cert': os.environ.get('BIGCHAINDB_DATABASE_CA_CERT'),
-    'certfile': os.environ.get('BIGCHAINDB_DATABASE_CERTFILE'),
-    'keyfile': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE'),
-    'keyfile_passphrase': os.environ.get('BIGCHAINDB_DATABASE_KEYFILE_PASSPHRASE'),
-    'crlfile': os.environ.get('BIGCHAINDB_DATABASE_CRLFILE')
-}
-_database_mongodb.update(_base_database_mongodb)
+_database_localmongodb.update(_base_database_localmongodb)
 
 _database_map = {
-    'mongodb': _database_mongodb,
-    'rethinkdb': _database_rethinkdb
+    'localmongodb': _database_localmongodb,
 }
 
 config = {
     'server': {
         # Note: this section supports all the Gunicorn settings:
         #       - http://docs.gunicorn.org/en/stable/settings.html
-        'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984',
+        'bind': 'localhost:9984',
         'loglevel': logging.getLevelName(
             log_config['handlers']['console']['level']).lower(),
-        'workers': None,  # if none, the value will be cpu_count * 2 + 1
+        'workers': None,  # if None, the value will be cpu_count * 2 + 1
     },
     'wsserver': {
-        'scheme': os.environ.get('BIGCHAINDB_WSSERVER_SCHEME') or 'ws',
-        'host': os.environ.get('BIGCHAINDB_WSSERVER_HOST') or 'localhost',
-        'port': int(os.environ.get('BIGCHAINDB_WSSERVER_PORT', 9985)),
+        'scheme': 'ws',
+        'host': 'localhost',
+        'port': 9985,
+        'advertised_scheme': 'ws',
+        'advertised_host': 'localhost',
+        'advertised_port': 9985,
     },
-    'database': _database_map[
-        os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb')
-    ],
-    'keypair': {
-        'public': None,
-        'private': None,
+    'tendermint': {
+        'host': 'localhost',
+        'port': 26657,
+        'version': 'v0.31.5',  # look for __tm_supported_versions__
     },
-    'keyring': [],
-    'backlog_reassign_delay': 120,
+    # FIXME: hardcoding to localmongodb for now
+    'database': _database_map['localmongodb'],
     'log': {
         'file': log_config['handlers']['file']['filename'],
         'error_file': log_config['handlers']['errors']['filename'],
@@ -95,14 +88,19 @@ config = {
         'fmt_logfile': log_config['formatters']['file']['format'],
         'granular_levels': {},
     },
-    'graphite': {
-        'host': os.environ.get('BIGCHAINDB_GRAPHITE_HOST', 'localhost'),
-    },
 }
 
 # We need to maintain a backup copy of the original config dict in case
 # the user wants to reconfigure the node. Check ``bigchaindb.config_utils``
 # for more info.
 _config = copy.deepcopy(config)
-from bigchaindb.core import Bigchain  # noqa
-from bigchaindb.version import __version__  # noqa
+from bigchaindb.common.transaction import Transaction  # noqa
+from bigchaindb import models                          # noqa
+from bigchaindb.upsert_validator import ValidatorElection  # noqa
+from bigchaindb.elections.vote import Vote  # noqa
+
+Transaction.register_type(Transaction.CREATE, models.Transaction)
+Transaction.register_type(Transaction.TRANSFER, models.Transaction)
+Transaction.register_type(ValidatorElection.OPERATION, ValidatorElection)
+Transaction.register_type(ChainMigrationElection.OPERATION, ChainMigrationElection)
+Transaction.register_type(Vote.OPERATION, Vote)

bigchaindb/backend/README.md

@@ -1,14 +1,20 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Backend Interfaces
 
 ## Structure
 
-- [`changefeed.py`](./changefeed.py): Changefeed-related interfaces
 - [`connection.py`](./connection.py): Database connection-related interfaces
 - [`query.py`](./query.py): Database query-related interfaces, dispatched through single-dispatch
 - [`schema.py`](./schema.py): Database setup and schema-related interfaces, dispatched through
   single-dispatch
 
-Built-in implementations (e.g. [RethinkDB's](./rethinkdb)) are provided in sub-directories and
+Built-in implementations (e.g. [MongoDB's](./localmongodb)) are provided in sub-directories and
 have their connection type's location exposed as `BACKENDS` in [`connection.py`](./connection.py).
 
 ## Single-Dispatched Interfaces

bigchaindb/backend/__init__.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Generic backend database interfaces expected by BigchainDB.
 
 The interfaces in this module allow BigchainDB to be agnostic about its
@@ -7,7 +12,6 @@ configuration or the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable.
 """
 
 # Include the backend interfaces
-from bigchaindb.backend import admin, changefeed, schema, query  # noqa
+from bigchaindb.backend import schema, query  # noqa
 
 from bigchaindb.backend.connection import connect  # noqa
-from bigchaindb.backend.changefeed import get_changefeed  # noqa

bigchaindb/backend/admin.py

@@ -1,34 +0,0 @@
-"""Database configuration functions."""
-from functools import singledispatch
-
-
-@singledispatch
-def get_config(connection, *, table):
-    raise NotImplementedError
-
-
-@singledispatch
-def reconfigure(connection, *, table, shards, replicas, **kwargs):
-    raise NotImplementedError
-
-
-@singledispatch
-def set_shards(connection, *, shards):
-    raise NotImplementedError
-
-
-@singledispatch
-def set_replicas(connection, *, replicas):
-    raise NotImplementedError
-
-
-@singledispatch
-def add_replicas(connection, replicas):
-    raise NotImplementedError('This command is specific to the '
-                              'MongoDB backend.')
-
-
-@singledispatch
-def remove_replicas(connection, replicas):
-    raise NotImplementedError('This command is specific to the '
-                              'MongoDB backend.')

bigchaindb/backend/changefeed.py

@@ -1,90 +0,0 @@
-"""Changefeed interfaces for backends."""
-
-from functools import singledispatch
-
-from multipipes import Node
-
-import bigchaindb
-
-
-class ChangeFeed(Node):
-    """Create a new changefeed.
-
-    It extends :class:`multipipes.Node` to make it pluggable in other
-    Pipelines instances, and makes usage of ``self.outqueue`` to output
-    the data.
-
-    A changefeed is a real time feed on inserts, updates, and deletes, and
-    is volatile. This class is a helper to create changefeeds. Moreover,
-    it provides a way to specify a ``prefeed`` of iterable data to output
-    before the actual changefeed.
-    """
-
-    INSERT = 1
-    DELETE = 2
-    UPDATE = 4
-
-    def __init__(self, table, operation, *, prefeed=None, connection=None):
-        """Create a new ChangeFeed.
-
-        Args:
-            table (str): name of the table to listen to for changes.
-            operation (int): can be ChangeFeed.INSERT, ChangeFeed.DELETE, or
-                ChangeFeed.UPDATE. Combining multiple operations is possible
-                with the bitwise ``|`` operator
-                (e.g. ``ChangeFeed.INSERT | ChangeFeed.UPDATE``)
-            prefeed (:class:`~collections.abc.Iterable`, optional): whatever
-                set of data you want to be published first.
-            connection (:class:`~bigchaindb.backend.connection.Connection`, optional):  # noqa
-                A connection to the database. If no connection is provided a
-                default connection will be created.
-        """
-
-        super().__init__(name='changefeed')
-        self.prefeed = prefeed if prefeed else []
-        self.table = table
-        self.operation = operation
-        if connection:
-            self.connection = connection
-        else:
-            self.connection = bigchaindb.backend.connect(
-                **bigchaindb.config['database'])
-
-    def run_forever(self):
-        """Main loop of the ``multipipes.Node``
-
-        This method is responsible for first feeding the prefeed to the
-        outqueue and after that starting the changefeed and recovering from any
-        errors that may occur in the backend.
-        """
-        raise NotImplementedError
-
-    def run_changefeed(self):
-        """Backend specific method to run the changefeed.
-
-        The changefeed is usually a backend cursor that is not closed when all
-        the results are exausted. Instead it remains open waiting for new
-        results.
-
-        This method should also filter each result based on the ``operation``
-        and put all matching results on the outqueue of ``multipipes.Node``.
-        """
-        raise NotImplementedError
-
-
-@singledispatch
-def get_changefeed(connection, table, operation, *, prefeed=None):
-    """Return a ChangeFeed.
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        table (str): name of the table to listen to for changes.
-        operation (int): can be ChangeFeed.INSERT, ChangeFeed.DELETE, or
-            ChangeFeed.UPDATE. Combining multiple operation is possible
-            with the bitwise ``|`` operator
-            (e.g. ``ChangeFeed.INSERT | ChangeFeed.UPDATE``)
-        prefeed (iterable): whatever set of data you want to be published
-            first.
-    """
-    raise NotImplementedError

bigchaindb/backend/connection.py

@@ -1,15 +1,19 @@
-from itertools import repeat
-from importlib import import_module
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 import logging
+from importlib import import_module
+from itertools import repeat
 
 import bigchaindb
-from bigchaindb.common.exceptions import ConfigurationError
 from bigchaindb.backend.exceptions import ConnectionError
-
+from bigchaindb.backend.utils import get_bigchaindb_config_value, get_bigchaindb_config_value_or_key_error
+from bigchaindb.common.exceptions import ConfigurationError
 
 BACKENDS = {
-    'mongodb': 'bigchaindb.backend.mongodb.connection.MongoDBConnection',
-    'rethinkdb': 'bigchaindb.backend.rethinkdb.connection.RethinkDBConnection'
+    'localmongodb': 'bigchaindb.backend.localmongodb.connection.LocalMongoDBConnection',
 }
 
 logger = logging.getLogger(__name__)
@@ -44,24 +48,28 @@ def connect(backend=None, host=None, port=None, name=None, max_tries=None,
             Authentication failure after connecting to the database.
     """
 
-    backend = backend or bigchaindb.config['database']['backend']
-    host = host or bigchaindb.config['database']['host']
-    port = port or bigchaindb.config['database']['port']
-    dbname = name or bigchaindb.config['database']['name']
+    backend = backend or get_bigchaindb_config_value_or_key_error('backend')
+    host = host or get_bigchaindb_config_value_or_key_error('host')
+    port = port or get_bigchaindb_config_value_or_key_error('port')
+    dbname = name or get_bigchaindb_config_value_or_key_error('name')
     # Not sure how to handle this here. This setting is only relevant for
     # mongodb.
     # I added **kwargs for both RethinkDBConnection and MongoDBConnection
     # to handle these these additional args. In case of RethinkDBConnection
     # it just does not do anything with it.
-    replicaset = replicaset or bigchaindb.config['database'].get('replicaset')
-    ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False)
-    login = login or bigchaindb.config['database'].get('login')
-    password = password or bigchaindb.config['database'].get('password')
-    ca_cert = ca_cert or bigchaindb.config['database'].get('ca_cert', None)
-    certfile = certfile or bigchaindb.config['database'].get('certfile', None)
-    keyfile = keyfile or bigchaindb.config['database'].get('keyfile', None)
-    keyfile_passphrase = keyfile_passphrase or bigchaindb.config['database'].get('keyfile_passphrase', None)
-    crlfile = crlfile or bigchaindb.config['database'].get('crlfile', None)
+    #
+    # UPD: RethinkDBConnection is not here anymore cause we no longer support RethinkDB.
+    # The problem described above might be reconsidered next time we introduce a backend,
+    # if it ever happens.
+    replicaset = replicaset or get_bigchaindb_config_value('replicaset')
+    ssl = ssl if ssl is not None else get_bigchaindb_config_value('ssl', False)
+    login = login or get_bigchaindb_config_value('login')
+    password = password or get_bigchaindb_config_value('password')
+    ca_cert = ca_cert or get_bigchaindb_config_value('ca_cert')
+    certfile = certfile or get_bigchaindb_config_value('certfile')
+    keyfile = keyfile or get_bigchaindb_config_value('keyfile')
+    keyfile_passphrase = keyfile_passphrase or get_bigchaindb_config_value('keyfile_passphrase', None)
+    crlfile = crlfile or get_bigchaindb_config_value('crlfile')
 
     try:
         module_name, _, class_name = BACKENDS[backend].rpartition('.')
@@ -110,7 +118,7 @@ class Connection:
         self.host = host or dbconf['host']
         self.port = port or dbconf['port']
         self.dbname = dbname or dbconf['name']
-        self.connection_timeout = connection_timeout if connection_timeout is not None\
+        self.connection_timeout = connection_timeout if connection_timeout is not None \
             else dbconf['connection_timeout']
         self.max_tries = max_tries if max_tries is not None else dbconf['max_tries']
         self.max_tries_counter = range(self.max_tries) if self.max_tries != 0 else repeat(0)

bigchaindb/backend/exceptions.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 from bigchaindb.exceptions import BigchainDBError
 
 

bigchaindb/backend/localmongodb/__init__.py

@@ -1,22 +1,28 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """MongoDB backend implementation.
 
 Contains a MongoDB-specific implementation of the
-:mod:`~bigchaindb.backend.changefeed`, :mod:`~bigchaindb.backend.query`, and
-:mod:`~bigchaindb.backend.schema` interfaces.
+:mod:`~bigchaindb.backend.schema` and :mod:`~bigchaindb.backend.query` interfaces.
 
 You can specify BigchainDB to use MongoDB as its database backend by either
-setting ``database.backend`` to ``'rethinkdb'`` in your configuration file, or
+setting ``database.backend`` to ``'localmongodb'`` in your configuration file, or
 setting the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable to
-``'rethinkdb'``.
+``'localmongodb'``.
+
+MongoDB is the default database backend for BigchainDB.
 
 If configured to use MongoDB, BigchainDB will automatically return instances
-of :class:`~bigchaindb.backend.rethinkdb.MongoDBConnection` for
+of :class:`~bigchaindb.backend.localmongodb.LocalMongoDBConnection` for
 :func:`~bigchaindb.backend.connection.connect` and dispatch calls of the
 generic backend interfaces to the implementations in this module.
 """
 
 # Register the single dispatched modules on import.
-from bigchaindb.backend.mongodb import admin, schema, query, changefeed  # noqa
+from bigchaindb.backend.localmongodb import schema, query # noqa
 
 # MongoDBConnection should always be accessed via
 # ``bigchaindb.backend.connect()``.

bigchaindb/backend/localmongodb/connection.py

@@ -0,0 +1,136 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+import logging
+from ssl import CERT_REQUIRED
+
+import pymongo
+
+from bigchaindb.backend.connection import Connection
+from bigchaindb.backend.exceptions import (DuplicateKeyError,
+                                           OperationError,
+                                           ConnectionError)
+from bigchaindb.backend.utils import get_bigchaindb_config_value
+from bigchaindb.common.exceptions import ConfigurationError
+from bigchaindb.utils import Lazy
+
+logger = logging.getLogger(__name__)
+
+
+class LocalMongoDBConnection(Connection):
+
+    def __init__(self, replicaset=None, ssl=None, login=None, password=None,
+                 ca_cert=None, certfile=None, keyfile=None,
+                 keyfile_passphrase=None, crlfile=None, **kwargs):
+        """Create a new Connection instance.
+
+        Args:
+            replicaset (str, optional): the name of the replica set to
+                                        connect to.
+            **kwargs: arbitrary keyword arguments provided by the
+                configuration's ``database`` settings
+        """
+
+        super().__init__(**kwargs)
+        self.replicaset = replicaset or get_bigchaindb_config_value('replicaset')
+        self.ssl = ssl if ssl is not None else get_bigchaindb_config_value('ssl', False)
+        self.login = login or get_bigchaindb_config_value('login')
+        self.password = password or get_bigchaindb_config_value('password')
+        self.ca_cert = ca_cert or get_bigchaindb_config_value('ca_cert')
+        self.certfile = certfile or get_bigchaindb_config_value('certfile')
+        self.keyfile = keyfile or get_bigchaindb_config_value('keyfile')
+        self.keyfile_passphrase = keyfile_passphrase or get_bigchaindb_config_value('keyfile_passphrase')
+        self.crlfile = crlfile or get_bigchaindb_config_value('crlfile')
+
+    @property
+    def db(self):
+        return self.conn[self.dbname]
+
+    def query(self):
+        return Lazy()
+
+    def collection(self, name):
+        """Return a lazy object that can be used to compose a query.
+
+        Args:
+            name (str): the name of the collection to query.
+        """
+        return self.query()[self.dbname][name]
+
+    def run(self, query):
+        try:
+            try:
+                return query.run(self.conn)
+            except pymongo.errors.AutoReconnect:
+                logger.warning('Lost connection to the database, '
+                               'retrying query.')
+                return query.run(self.conn)
+        except pymongo.errors.AutoReconnect as exc:
+            raise ConnectionError from exc
+        except pymongo.errors.DuplicateKeyError as exc:
+            raise DuplicateKeyError from exc
+        except pymongo.errors.OperationFailure as exc:
+            print(f'DETAILS: {exc.details}')
+            raise OperationError from exc
+
+    def _connect(self):
+        """Try to connect to the database.
+
+        Raises:
+            :exc:`~ConnectionError`: If the connection to the database
+                fails.
+            :exc:`~AuthenticationError`: If there is a OperationFailure due to
+                Authentication failure after connecting to the database.
+            :exc:`~ConfigurationError`: If there is a ConfigurationError while
+                connecting to the database.
+        """
+
+        try:
+            # FYI: the connection process might raise a
+            # `ServerSelectionTimeoutError`, that is a subclass of
+            # `ConnectionFailure`.
+            # The presence of ca_cert, certfile, keyfile, crlfile implies the
+            # use of certificates for TLS connectivity.
+            if self.ca_cert is None or self.certfile is None or \
+                    self.keyfile is None or self.crlfile is None:
+                client = pymongo.MongoClient(self.host,
+                                             self.port,
+                                             replicaset=self.replicaset,
+                                             serverselectiontimeoutms=self.connection_timeout,
+                                             ssl=self.ssl,
+                                             **MONGO_OPTS)
+                if self.login is not None and self.password is not None:
+                    client[self.dbname].authenticate(self.login, self.password)
+            else:
+                logger.info('Connecting to MongoDB over TLS/SSL...')
+                client = pymongo.MongoClient(self.host,
+                                             self.port,
+                                             replicaset=self.replicaset,
+                                             serverselectiontimeoutms=self.connection_timeout,
+                                             ssl=self.ssl,
+                                             ssl_ca_certs=self.ca_cert,
+                                             ssl_certfile=self.certfile,
+                                             ssl_keyfile=self.keyfile,
+                                             ssl_pem_passphrase=self.keyfile_passphrase,
+                                             ssl_crlfile=self.crlfile,
+                                             ssl_cert_reqs=CERT_REQUIRED,
+                                             **MONGO_OPTS)
+                if self.login is not None:
+                    client[self.dbname].authenticate(self.login,
+                                                     mechanism='MONGODB-X509')
+
+            return client
+
+        except (pymongo.errors.ConnectionFailure,
+                pymongo.errors.OperationFailure) as exc:
+            logger.info('Exception in _connect(): {}'.format(exc))
+            raise ConnectionError(str(exc)) from exc
+        except pymongo.errors.ConfigurationError as exc:
+            raise ConfigurationError from exc
+
+
+MONGO_OPTS = {
+    'socketTimeoutMS': 20000,
+}

bigchaindb/backend/localmongodb/query.py

@@ -0,0 +1,377 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+"""Query implementation for MongoDB"""
+
+from pymongo import DESCENDING
+
+from bigchaindb import backend
+from bigchaindb.backend.exceptions import DuplicateKeyError
+from bigchaindb.backend.utils import module_dispatch_registrar
+from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection
+from bigchaindb.common.transaction import Transaction
+
+register_query = module_dispatch_registrar(backend.query)
+
+
+@register_query(LocalMongoDBConnection)
+def store_transactions(conn, signed_transactions):
+    return conn.run(conn.collection('transactions')
+                    .insert_many(signed_transactions))
+
+
+@register_query(LocalMongoDBConnection)
+def get_transaction(conn, transaction_id):
+    return conn.run(
+        conn.collection('transactions')
+        .find_one({'id': transaction_id}, {'_id': 0}))
+
+
+@register_query(LocalMongoDBConnection)
+def get_transactions(conn, transaction_ids):
+    try:
+        return conn.run(
+            conn.collection('transactions')
+            .find({'id': {'$in': transaction_ids}},
+                  projection={'_id': False}))
+    except IndexError:
+        pass
+
+
+@register_query(LocalMongoDBConnection)
+def store_metadatas(conn, metadata):
+    return conn.run(
+        conn.collection('metadata')
+        .insert_many(metadata, ordered=False))
+
+
+@register_query(LocalMongoDBConnection)
+def get_metadata(conn, transaction_ids):
+    return conn.run(
+        conn.collection('metadata')
+        .find({'id': {'$in': transaction_ids}},
+              projection={'_id': False}))
+
+
+@register_query(LocalMongoDBConnection)
+def store_asset(conn, asset):
+    try:
+        return conn.run(
+            conn.collection('assets')
+            .insert_one(asset))
+    except DuplicateKeyError:
+        pass
+
+
+@register_query(LocalMongoDBConnection)
+def store_assets(conn, assets):
+    return conn.run(
+        conn.collection('assets')
+        .insert_many(assets, ordered=False))
+
+
+@register_query(LocalMongoDBConnection)
+def get_asset(conn, asset_id):
+    try:
+        return conn.run(
+            conn.collection('assets')
+            .find_one({'id': asset_id}, {'_id': 0, 'id': 0}))
+    except IndexError:
+        pass
+
+
+@register_query(LocalMongoDBConnection)
+def get_assets(conn, asset_ids):
+    return conn.run(
+        conn.collection('assets')
+        .find({'id': {'$in': asset_ids}},
+              projection={'_id': False}))
+
+
+@register_query(LocalMongoDBConnection)
+def get_spent(conn, transaction_id, output):
+    query = {'inputs':
+             {'$elemMatch':
+              {'$and': [{'fulfills.transaction_id': transaction_id},
+                        {'fulfills.output_index': output}]}}}
+
+    return conn.run(
+        conn.collection('transactions')
+            .find(query, {'_id': 0}))
+
+
+@register_query(LocalMongoDBConnection)
+def get_latest_block(conn):
+    return conn.run(
+        conn.collection('blocks')
+        .find_one(projection={'_id': False},
+                  sort=[('height', DESCENDING)]))
+
+
+@register_query(LocalMongoDBConnection)
+def store_block(conn, block):
+    try:
+        return conn.run(
+            conn.collection('blocks')
+            .insert_one(block))
+    except DuplicateKeyError:
+        pass
+
+
+@register_query(LocalMongoDBConnection)
+def get_txids_filtered(conn, asset_id, operation=None, last_tx=None):
+
+    match = {
+        Transaction.CREATE: {'operation': 'CREATE', 'id': asset_id},
+        Transaction.TRANSFER: {'operation': 'TRANSFER', 'asset.id': asset_id},
+        None: {'$or': [{'asset.id': asset_id}, {'id': asset_id}]},
+    }[operation]
+
+    cursor = conn.run(conn.collection('transactions').find(match))
+
+    if last_tx:
+        cursor = cursor.sort([('$natural', DESCENDING)]).limit(1)
+
+    return (elem['id'] for elem in cursor)
+
+
+@register_query(LocalMongoDBConnection)
+def text_search(conn, search, *, language='english', case_sensitive=False,
+                diacritic_sensitive=False, text_score=False, limit=0, table='assets'):
+    cursor = conn.run(
+        conn.collection(table)
+        .find({'$text': {
+                '$search': search,
+                '$language': language,
+                '$caseSensitive': case_sensitive,
+                '$diacriticSensitive': diacritic_sensitive}},
+              {'score': {'$meta': 'textScore'}, '_id': False})
+        .sort([('score', {'$meta': 'textScore'})])
+        .limit(limit))
+
+    if text_score:
+        return cursor
+
+    return (_remove_text_score(obj) for obj in cursor)
+
+
+def _remove_text_score(asset):
+    asset.pop('score', None)
+    return asset
+
+
+@register_query(LocalMongoDBConnection)
+def get_owned_ids(conn, owner):
+    cursor = conn.run(
+        conn.collection('transactions').aggregate([
+            {'$match': {'outputs.public_keys': owner}},
+            {'$project': {'_id': False}}
+        ]))
+    return cursor
+
+
+@register_query(LocalMongoDBConnection)
+def get_spending_transactions(conn, inputs):
+    transaction_ids = [i['transaction_id'] for i in inputs]
+    output_indexes = [i['output_index'] for i in inputs]
+    query = {'inputs':
+             {'$elemMatch':
+              {'$and':
+               [
+                   {'fulfills.transaction_id': {'$in': transaction_ids}},
+                   {'fulfills.output_index': {'$in': output_indexes}}
+               ]}}}
+
+    cursor = conn.run(
+        conn.collection('transactions').find(query, {'_id': False}))
+    return cursor
+
+
+@register_query(LocalMongoDBConnection)
+def get_block(conn, block_id):
+    return conn.run(
+        conn.collection('blocks')
+        .find_one({'height': block_id},
+                  projection={'_id': False}))
+
+
+@register_query(LocalMongoDBConnection)
+def get_block_with_transaction(conn, txid):
+    return conn.run(
+        conn.collection('blocks')
+        .find({'transactions': txid},
+              projection={'_id': False, 'height': True}))
+
+
+@register_query(LocalMongoDBConnection)
+def delete_transactions(conn, txn_ids):
+    conn.run(conn.collection('assets').delete_many({'id': {'$in': txn_ids}}))
+    conn.run(conn.collection('metadata').delete_many({'id': {'$in': txn_ids}}))
+    conn.run(conn.collection('transactions').delete_many({'id': {'$in': txn_ids}}))
+
+
+@register_query(LocalMongoDBConnection)
+def store_unspent_outputs(conn, *unspent_outputs):
+    if unspent_outputs:
+        try:
+            return conn.run(
+                conn.collection('utxos').insert_many(
+                    unspent_outputs,
+                    ordered=False,
+                )
+            )
+        except DuplicateKeyError:
+            # TODO log warning at least
+            pass
+
+
+@register_query(LocalMongoDBConnection)
+def delete_unspent_outputs(conn, *unspent_outputs):
+    if unspent_outputs:
+        return conn.run(
+            conn.collection('utxos').delete_many({
+                '$or': [{
+                    '$and': [
+                        {'transaction_id': unspent_output['transaction_id']},
+                        {'output_index': unspent_output['output_index']},
+                    ],
+                } for unspent_output in unspent_outputs]
+            })
+        )
+
+
+@register_query(LocalMongoDBConnection)
+def get_unspent_outputs(conn, *, query=None):
+    if query is None:
+        query = {}
+    return conn.run(conn.collection('utxos').find(query,
+                                                  projection={'_id': False}))
+
+
+@register_query(LocalMongoDBConnection)
+def store_pre_commit_state(conn, state):
+    return conn.run(
+        conn.collection('pre_commit')
+        .replace_one({}, state, upsert=True)
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def get_pre_commit_state(conn):
+    return conn.run(conn.collection('pre_commit').find_one())
+
+
+@register_query(LocalMongoDBConnection)
+def store_validator_set(conn, validators_update):
+    height = validators_update['height']
+    return conn.run(
+        conn.collection('validators').replace_one(
+            {'height': height},
+            validators_update,
+            upsert=True
+        )
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def delete_validator_set(conn, height):
+    return conn.run(
+        conn.collection('validators').delete_many({'height': height})
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def store_election(conn, election_id, height, is_concluded):
+    return conn.run(
+        conn.collection('elections').replace_one(
+            {'election_id': election_id,
+             'height': height},
+            {'election_id': election_id,
+             'height': height,
+             'is_concluded': is_concluded},
+            upsert=True,
+        )
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def store_elections(conn, elections):
+    return conn.run(
+        conn.collection('elections').insert_many(elections)
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def delete_elections(conn, height):
+    return conn.run(
+        conn.collection('elections').delete_many({'height': height})
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def get_validator_set(conn, height=None):
+    query = {}
+    if height is not None:
+        query = {'height': {'$lte': height}}
+
+    cursor = conn.run(
+        conn.collection('validators')
+        .find(query, projection={'_id': False})
+        .sort([('height', DESCENDING)])
+        .limit(1)
+    )
+
+    return next(cursor, None)
+
+
+@register_query(LocalMongoDBConnection)
+def get_election(conn, election_id):
+    query = {'election_id': election_id}
+
+    return conn.run(
+        conn.collection('elections')
+        .find_one(query, projection={'_id': False},
+                  sort=[('height', DESCENDING)])
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def get_asset_tokens_for_public_key(conn, asset_id, public_key):
+    query = {'outputs.public_keys': [public_key],
+             'asset.id': asset_id}
+
+    cursor = conn.run(
+        conn.collection('transactions').aggregate([
+            {'$match': query},
+            {'$project': {'_id': False}}
+        ]))
+    return cursor
+
+
+@register_query(LocalMongoDBConnection)
+def store_abci_chain(conn, height, chain_id, is_synced=True):
+    return conn.run(
+        conn.collection('abci_chains').replace_one(
+            {'height': height},
+            {'height': height, 'chain_id': chain_id,
+             'is_synced': is_synced},
+            upsert=True,
+        )
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def delete_abci_chain(conn, height):
+    return conn.run(
+        conn.collection('abci_chains').delete_many({'height': height})
+    )
+
+
+@register_query(LocalMongoDBConnection)
+def get_latest_abci_chain(conn):
+    return conn.run(
+        conn.collection('abci_chains')
+            .find_one(projection={'_id': False}, sort=[('height', DESCENDING)])
+    )

bigchaindb/backend/localmongodb/schema.py

@@ -0,0 +1,90 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+"""Utils to initialize and drop the database."""
+
+import logging
+
+from pymongo import ASCENDING, DESCENDING, TEXT
+from pymongo.errors import CollectionInvalid
+
+from bigchaindb import backend
+from bigchaindb.backend.utils import module_dispatch_registrar
+from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection
+
+
+logger = logging.getLogger(__name__)
+register_schema = module_dispatch_registrar(backend.schema)
+
+
+INDEXES = {
+    'transactions': [
+        ('id', dict(unique=True, name='transaction_id')),
+        ('asset.id', dict(name='asset_id')),
+        ('outputs.public_keys', dict(name='outputs')),
+        ([('inputs.fulfills.transaction_id', ASCENDING),
+          ('inputs.fulfills.output_index', ASCENDING)], dict(name='inputs')),
+    ],
+    'assets': [
+        ('id', dict(name='asset_id', unique=True)),
+        ([('$**', TEXT)], dict(name='text')),
+    ],
+    'blocks': [
+        ([('height', DESCENDING)], dict(name='height', unique=True)),
+    ],
+    'metadata': [
+        ('id', dict(name='transaction_id', unique=True)),
+        ([('$**', TEXT)], dict(name='text')),
+    ],
+    'utxos': [
+        ([('transaction_id', ASCENDING),
+          ('output_index', ASCENDING)], dict(name='utxo', unique=True)),
+    ],
+    'pre_commit': [
+        ('height', dict(name='height', unique=True)),
+    ],
+    'elections': [
+        ([('height', DESCENDING), ('election_id', ASCENDING)],
+         dict(name='election_id_height', unique=True)),
+    ],
+    'validators': [
+        ('height', dict(name='height', unique=True)),
+    ],
+    'abci_chains': [
+        ('height', dict(name='height', unique=True)),
+        ('chain_id', dict(name='chain_id', unique=True)),
+    ],
+}
+
+
+@register_schema(LocalMongoDBConnection)
+def create_database(conn, dbname):
+    logger.info('Create database `%s`.', dbname)
+    # TODO: read and write concerns can be declared here
+    conn.conn.get_database(dbname)
+
+
+@register_schema(LocalMongoDBConnection)
+def create_tables(conn, dbname):
+    for table_name in backend.schema.TABLES:
+        # create the table
+        # TODO: read and write concerns can be declared here
+        try:
+            logger.info(f'Create `{table_name}` table.')
+            conn.conn[dbname].create_collection(table_name)
+        except CollectionInvalid:
+            logger.info(f'Collection {table_name} already exists.')
+        create_indexes(conn, dbname, table_name, INDEXES[table_name])
+
+
+def create_indexes(conn, dbname, collection, indexes):
+    logger.info(f'Ensure secondary indexes for `{collection}`.')
+    for fields, kwargs in indexes:
+        conn.conn[dbname][collection].create_index(fields, **kwargs)
+
+
+@register_schema(LocalMongoDBConnection)
+def drop_database(conn, dbname):
+    conn.conn.drop_database(dbname)

bigchaindb/backend/mongodb/admin.py

@@ -1,86 +0,0 @@
-"""Database configuration functions."""
-import logging
-
-from pymongo.errors import OperationFailure
-
-from bigchaindb.backend import admin
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.exceptions import OperationError
-from bigchaindb.backend.mongodb.connection import MongoDBConnection
-
-logger = logging.getLogger(__name__)
-
-register_admin = module_dispatch_registrar(admin)
-
-
-@register_admin(MongoDBConnection)
-def add_replicas(connection, replicas):
-    """Add a set of replicas to the replicaset
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        replicas (:obj:`list` of :obj:`str`): replica addresses in the
-            form "hostname:port".
-
-    Raises:
-        OperationError: If the reconfiguration fails due to a MongoDB
-            :exc:`OperationFailure`
-    """
-    # get current configuration
-    conf = connection.conn.admin.command('replSetGetConfig')
-
-    # MongoDB does not automatically add an id for the members so we need
-    # to choose one that does not exist yet. The safest way is to use
-    # incrementing ids, so we first check what is the highest id already in
-    # the set and continue from there.
-    cur_id = max([member['_id'] for member in conf['config']['members']])
-
-    # add the nodes to the members list of the replica set
-    for replica in replicas:
-        cur_id += 1
-        conf['config']['members'].append({'_id': cur_id, 'host': replica})
-
-    # increase the configuration version number
-    # when reconfiguring, mongodb expects a version number higher than the one
-    # it currently has
-    conf['config']['version'] += 1
-
-    # apply new configuration
-    try:
-        connection.conn.admin.command('replSetReconfig', conf['config'])
-    except OperationFailure as exc:
-        raise OperationError(exc.details['errmsg'])
-
-
-@register_admin(MongoDBConnection)
-def remove_replicas(connection, replicas):
-    """Remove a set of replicas from the replicaset
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        replicas (:obj:`list` of :obj:`str`): replica addresses in the
-            form "hostname:port".
-
-    Raises:
-        OperationError: If the reconfiguration fails due to a MongoDB
-            :exc:`OperationFailure`
-    """
-    # get the current configuration
-    conf = connection.conn.admin.command('replSetGetConfig')
-
-    # remove the nodes from the members list in the replica set
-    conf['config']['members'] = list(
-        filter(lambda member: member['host'] not in replicas,
-               conf['config']['members'])
-    )
-
-    # increase the configuration version number
-    conf['config']['version'] += 1
-
-    # apply new configuration
-    try:
-        connection.conn.admin.command('replSetReconfig', conf['config'])
-    except OperationFailure as exc:
-        raise OperationError(exc.details['errmsg'])

bigchaindb/backend/mongodb/changefeed.py

@@ -1,111 +0,0 @@
-import logging
-import time
-
-import pymongo
-
-from bigchaindb import backend
-from bigchaindb.backend.changefeed import ChangeFeed
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.mongodb.connection import MongoDBConnection
-from bigchaindb.backend.exceptions import BackendError
-
-
-logger = logging.getLogger(__name__)
-register_changefeed = module_dispatch_registrar(backend.changefeed)
-
-
-class MongoDBChangeFeed(ChangeFeed):
-    """This class implements a MongoDB changefeed as a multipipes Node.
-
-    We emulate the behaviour of the RethinkDB changefeed by using a tailable
-    cursor that listens for events on the oplog.
-    """
-    def run_forever(self):
-        for element in self.prefeed:
-            self.outqueue.put(element)
-
-        table = self.table
-        dbname = self.connection.dbname
-
-        # last timestamp in the oplog. We only care for operations happening
-        # in the future.
-        last_ts = self.connection.run(
-            self.connection.query().local.oplog.rs.find()
-            .sort('$natural', pymongo.DESCENDING).limit(1)
-            .next()['ts'])
-
-        for record in run_changefeed(self.connection, table, last_ts):
-
-            is_insert = record['op'] == 'i'
-            is_delete = record['op'] == 'd'
-            is_update = record['op'] == 'u'
-
-            # mongodb documents uses the `_id` for the primary key.
-            # We are not using this field at this point and we need to
-            # remove it to prevent problems with schema validation.
-            # See https://github.com/bigchaindb/bigchaindb/issues/992
-            if is_insert and (self.operation & ChangeFeed.INSERT):
-                record['o'].pop('_id', None)
-                self.outqueue.put(record['o'])
-            elif is_delete and (self.operation & ChangeFeed.DELETE):
-                # on delete it only returns the id of the document
-                self.outqueue.put(record['o'])
-            elif is_update and (self.operation & ChangeFeed.UPDATE):
-                # the oplog entry for updates only returns the update
-                # operations to apply to the document and not the
-                # document itself. So here we first read the document
-                # and then return it.
-                doc = self.connection.conn[dbname][table].find_one(
-                    {'_id': record['o2']['_id']},
-                    {'_id': False}
-                )
-                self.outqueue.put(doc)
-
-            logger.debug('Record in changefeed: %s:%s', table, record['op'])
-
-
-@register_changefeed(MongoDBConnection)
-def get_changefeed(connection, table, operation, *, prefeed=None):
-    """Return a MongoDB changefeed.
-
-    Returns:
-        An instance of
-        :class:`~bigchaindb.backend.mongodb.MongoDBChangeFeed`.
-    """
-
-    return MongoDBChangeFeed(table, operation, prefeed=prefeed,
-                             connection=connection)
-
-
-_FEED_STOP = False
-"""If it's True then the changefeed will return when there are no more items.
-"""
-
-
-def run_changefeed(conn, table, last_ts):
-    """Encapsulate operational logic of tailing changefeed from MongoDB
-    """
-    while True:
-        try:
-            # XXX: hack to force reconnection, in case the connection
-            # is lost while waiting on the cursor. See #1154.
-            conn._conn = None
-            namespace = conn.dbname + '.' + table
-            query = conn.query().local.oplog.rs.find(
-                {'ns': namespace, 'ts': {'$gt': last_ts}},
-                {'o._id': False},
-                cursor_type=pymongo.CursorType.TAILABLE_AWAIT
-            )
-            cursor = conn.run(query)
-            logging.debug('Tailing oplog at %s/%s', namespace, last_ts)
-            while cursor.alive:
-                try:
-                    record = cursor.next()
-                    yield record
-                    last_ts = record['ts']
-                except StopIteration:
-                    if _FEED_STOP:
-                        return
-        except (BackendError, pymongo.errors.ConnectionFailure):
-            logger.exception('Lost connection while tailing oplog, retrying')
-            time.sleep(1)

bigchaindb/backend/mongodb/connection.py

@@ -1,258 +0,0 @@
-import time
-import logging
-from ssl import CERT_REQUIRED
-
-import pymongo
-
-import bigchaindb
-from bigchaindb.utils import Lazy
-from bigchaindb.common.exceptions import ConfigurationError
-from bigchaindb.backend.exceptions import (DuplicateKeyError,
-                                           OperationError,
-                                           ConnectionError)
-from bigchaindb.backend.connection import Connection
-
-logger = logging.getLogger(__name__)
-
-
-class MongoDBConnection(Connection):
-
-    def __init__(self, replicaset=None, ssl=None, login=None, password=None,
-                 ca_cert=None, certfile=None, keyfile=None,
-                 keyfile_passphrase=None, crlfile=None, **kwargs):
-
-        """Create a new Connection instance.
-
-        Args:
-            replicaset (str, optional): the name of the replica set to
-                                        connect to.
-            **kwargs: arbitrary keyword arguments provided by the
-                configuration's ``database`` settings
-        """
-
-        super().__init__(**kwargs)
-        self.replicaset = replicaset or bigchaindb.config['database']['replicaset']
-        self.ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False)
-        self.login = login or bigchaindb.config['database'].get('login')
-        self.password = password or bigchaindb.config['database'].get('password')
-        self.ca_cert = ca_cert or bigchaindb.config['database'].get('ca_cert', None)
-        self.certfile = certfile or bigchaindb.config['database'].get('certfile', None)
-        self.keyfile = keyfile or bigchaindb.config['database'].get('keyfile', None)
-        self.keyfile_passphrase = keyfile_passphrase or bigchaindb.config['database'].get('keyfile_passphrase', None)
-        self.crlfile = crlfile or bigchaindb.config['database'].get('crlfile', None)
-
-    @property
-    def db(self):
-        return self.conn[self.dbname]
-
-    def query(self):
-        return Lazy()
-
-    def collection(self, name):
-        """Return a lazy object that can be used to compose a query.
-
-        Args:
-            name (str): the name of the collection to query.
-        """
-        return self.query()[self.dbname][name]
-
-    def run(self, query):
-        try:
-            try:
-                return query.run(self.conn)
-            except pymongo.errors.AutoReconnect as exc:
-                logger.warning('Lost connection to the database, '
-                               'retrying query.')
-                return query.run(self.conn)
-        except pymongo.errors.AutoReconnect as exc:
-            raise ConnectionError from exc
-        except pymongo.errors.DuplicateKeyError as exc:
-            raise DuplicateKeyError from exc
-        except pymongo.errors.OperationFailure as exc:
-            raise OperationError from exc
-
-    def _connect(self):
-        """Try to connect to the database.
-
-        Raises:
-            :exc:`~ConnectionError`: If the connection to the database
-                fails.
-            :exc:`~AuthenticationError`: If there is a OperationFailure due to
-                Authentication failure after connecting to the database.
-            :exc:`~ConfigurationError`: If there is a ConfigurationError while
-                connecting to the database.
-        """
-
-        try:
-            # we should only return a connection if the replica set is
-            # initialized. initialize_replica_set will check if the
-            # replica set is initialized else it will initialize it.
-            initialize_replica_set(self.host,
-                                   self.port,
-                                   self.connection_timeout,
-                                   self.dbname,
-                                   self.ssl,
-                                   self.login,
-                                   self.password,
-                                   self.ca_cert,
-                                   self.certfile,
-                                   self.keyfile,
-                                   self.keyfile_passphrase,
-                                   self.crlfile)
-
-            # FYI: the connection process might raise a
-            # `ServerSelectionTimeoutError`, that is a subclass of
-            # `ConnectionFailure`.
-            # The presence of ca_cert, certfile, keyfile, crlfile implies the
-            # use of certificates for TLS connectivity.
-            if self.ca_cert is None or self.certfile is None or \
-                    self.keyfile is None or self.crlfile is None:
-                client = pymongo.MongoClient(self.host,
-                                             self.port,
-                                             replicaset=self.replicaset,
-                                             serverselectiontimeoutms=self.connection_timeout,
-                                             ssl=self.ssl)
-                if self.login is not None and self.password is not None:
-                    client[self.dbname].authenticate(self.login, self.password)
-            else:
-                logger.info('Connecting to MongoDB over TLS/SSL...')
-                client = pymongo.MongoClient(self.host,
-                                             self.port,
-                                             replicaset=self.replicaset,
-                                             serverselectiontimeoutms=self.connection_timeout,
-                                             ssl=self.ssl,
-                                             ssl_ca_certs=self.ca_cert,
-                                             ssl_certfile=self.certfile,
-                                             ssl_keyfile=self.keyfile,
-                                             ssl_pem_passphrase=self.keyfile_passphrase,
-                                             ssl_crlfile=self.crlfile,
-                                             ssl_cert_reqs=CERT_REQUIRED)
-                if self.login is not None:
-                    client[self.dbname].authenticate(self.login,
-                                                     mechanism='MONGODB-X509')
-
-            return client
-
-        # `initialize_replica_set` might raise `ConnectionFailure`,
-        # `OperationFailure` or `ConfigurationError`.
-        except (pymongo.errors.ConnectionFailure,
-                pymongo.errors.OperationFailure) as exc:
-            logger.info('Exception in _connect(): {}'.format(exc))
-            raise ConnectionError(str(exc)) from exc
-        except pymongo.errors.ConfigurationError as exc:
-            raise ConfigurationError from exc
-
-
-def initialize_replica_set(host, port, connection_timeout, dbname, ssl, login,
-                           password, ca_cert, certfile, keyfile,
-                           keyfile_passphrase, crlfile):
-    """Initialize a replica set. If already initialized skip."""
-
-    # Setup a MongoDB connection
-    # The reason we do this instead of `backend.connect` is that
-    # `backend.connect` will connect you to a replica set but this fails if
-    # you try to connect to a replica set that is not yet initialized
-    try:
-        # The presence of ca_cert, certfile, keyfile, crlfile implies the
-        # use of certificates for TLS connectivity.
-        if ca_cert is None or certfile is None or keyfile is None or \
-                crlfile is None:
-            conn = pymongo.MongoClient(host,
-                                       port,
-                                       serverselectiontimeoutms=connection_timeout,
-                                       ssl=ssl)
-            if login is not None and password is not None:
-                conn[dbname].authenticate(login, password)
-        else:
-            logger.info('Connecting to MongoDB over TLS/SSL...')
-            conn = pymongo.MongoClient(host,
-                                       port,
-                                       serverselectiontimeoutms=connection_timeout,
-                                       ssl=ssl,
-                                       ssl_ca_certs=ca_cert,
-                                       ssl_certfile=certfile,
-                                       ssl_keyfile=keyfile,
-                                       ssl_pem_passphrase=keyfile_passphrase,
-                                       ssl_crlfile=crlfile,
-                                       ssl_cert_reqs=CERT_REQUIRED)
-            if login is not None:
-                logger.info('Authenticating to the database...')
-                conn[dbname].authenticate(login, mechanism='MONGODB-X509')
-
-    except (pymongo.errors.ConnectionFailure,
-            pymongo.errors.OperationFailure) as exc:
-        logger.info('Exception in _connect(): {}'.format(exc))
-        raise ConnectionError(str(exc)) from exc
-    except pymongo.errors.ConfigurationError as exc:
-        raise ConfigurationError from exc
-
-    _check_replica_set(conn)
-    host = '{}:{}'.format(bigchaindb.config['database']['host'],
-                          bigchaindb.config['database']['port'])
-    config = {'_id': bigchaindb.config['database']['replicaset'],
-              'members': [{'_id': 0, 'host': host}]}
-
-    try:
-        conn.admin.command('replSetInitiate', config)
-    except pymongo.errors.OperationFailure as exc_info:
-        if exc_info.details['codeName'] == 'AlreadyInitialized':
-            return
-        raise
-    else:
-        _wait_for_replica_set_initialization(conn)
-        logger.info('Initialized replica set')
-    finally:
-        if conn is not None:
-            logger.info('Closing initial connection to MongoDB')
-            conn.close()
-
-
-def _check_replica_set(conn):
-    """Checks if the replSet option was enabled either through the command
-       line option or config file and if it matches the one provided by
-       bigchaindb configuration.
-
-       Note:
-           The setting we are looking for will have a different name depending
-           if it was set by the config file (`replSetName`) or by command
-           line arguments (`replSet`).
-
-        Raise:
-            :exc:`~ConfigurationError`: If mongod was not started with the
-            replSet option.
-    """
-    options = conn.admin.command('getCmdLineOpts')
-    try:
-        repl_opts = options['parsed']['replication']
-        repl_set_name = repl_opts.get('replSetName', repl_opts.get('replSet'))
-    except KeyError:
-        raise ConfigurationError('mongod was not started with'
-                                 ' the replSet option.')
-
-    bdb_repl_set_name = bigchaindb.config['database']['replicaset']
-    if repl_set_name != bdb_repl_set_name:
-        raise ConfigurationError('The replicaset configuration of '
-                                 'bigchaindb (`{}`) needs to match '
-                                 'the replica set name from MongoDB'
-                                 ' (`{}`)'.format(bdb_repl_set_name,
-                                                  repl_set_name))
-
-
-def _wait_for_replica_set_initialization(conn):
-    """Wait for a replica set to finish initialization.
-
-    If a replica set is being initialized for the first time it takes some
-    time. Nodes need to discover each other and an election needs to take
-    place. During this time the database is not writable so we need to wait
-    before continuing with the rest of the initialization
-    """
-
-    # I did not find a better way to do this for now.
-    # To check if the database is ready we will poll the mongodb logs until
-    # we find the line that says the database is ready
-    logger.info('Waiting for mongodb replica set initialization')
-    while True:
-        logs = conn.admin.command('getLog', 'rs')['log']
-        if any('database writes are now permitted' in line for line in logs):
-            return
-        time.sleep(0.1)

bigchaindb/backend/mongodb/query.py

@@ -1,371 +0,0 @@
-"""Query implementation for MongoDB"""
-
-from time import time
-
-from pymongo import ReturnDocument
-
-from bigchaindb import backend
-from bigchaindb.backend.mongodb.changefeed import run_changefeed
-from bigchaindb.common.exceptions import CyclicBlockchainError
-from bigchaindb.common.transaction import Transaction
-from bigchaindb.backend.exceptions import DuplicateKeyError, OperationError
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.mongodb.connection import MongoDBConnection
-
-
-register_query = module_dispatch_registrar(backend.query)
-
-
-@register_query(MongoDBConnection)
-def write_transaction(conn, signed_transaction):
-    try:
-        return conn.run(
-            conn.collection('backlog')
-            .insert_one(signed_transaction))
-    except DuplicateKeyError:
-        return
-
-
-@register_query(MongoDBConnection)
-def update_transaction(conn, transaction_id, doc):
-    # with mongodb we need to add update operators to the doc
-    doc = {'$set': doc}
-    return conn.run(
-        conn.collection('backlog')
-        .find_one_and_update(
-            {'id': transaction_id},
-            doc,
-            return_document=ReturnDocument.AFTER))
-
-
-@register_query(MongoDBConnection)
-def delete_transaction(conn, *transaction_id):
-    return conn.run(
-        conn.collection('backlog')
-        .delete_many({'id': {'$in': transaction_id}}))
-
-
-@register_query(MongoDBConnection)
-def get_stale_transactions(conn, reassign_delay):
-    return conn.run(
-        conn.collection('backlog')
-        .find({'assignment_timestamp': {'$lt': time() - reassign_delay}},
-              projection={'_id': False}))
-
-
-@register_query(MongoDBConnection)
-def get_transaction_from_block(conn, transaction_id, block_id):
-    try:
-        return conn.run(
-            conn.collection('bigchain')
-            .aggregate([
-                {'$match': {'id': block_id}},
-                {'$project': {
-                    'block.transactions': {
-                        '$filter': {
-                            'input': '$block.transactions',
-                            'as': 'transaction',
-                            'cond': {
-                                '$eq': ['$$transaction.id', transaction_id]
-                                }
-                            }
-                        }
-                    }}])
-            .next()['block']['transactions']
-            .pop())
-    except (StopIteration, IndexError):
-        # StopIteration is raised if the block was not found
-        # IndexError is returned if the block is found but no transactions
-        # match
-        return
-
-
-@register_query(MongoDBConnection)
-def get_transaction_from_backlog(conn, transaction_id):
-    return conn.run(
-        conn.collection('backlog')
-        .find_one({'id': transaction_id},
-                  projection={'_id': False,
-                              'assignee': False,
-                              'assignment_timestamp': False}))
-
-
-@register_query(MongoDBConnection)
-def get_blocks_status_from_transaction(conn, transaction_id):
-    return conn.run(
-        conn.collection('bigchain')
-        .find({'block.transactions.id': transaction_id},
-              projection=['id', 'block.voters']))
-
-
-@register_query(MongoDBConnection)
-def get_txids_filtered(conn, asset_id, operation=None):
-    match_create = {
-        'block.transactions.operation': 'CREATE',
-        'block.transactions.id': asset_id
-    }
-    match_transfer = {
-        'block.transactions.operation': 'TRANSFER',
-        'block.transactions.asset.id': asset_id
-    }
-
-    if operation == Transaction.CREATE:
-        match = match_create
-    elif operation == Transaction.TRANSFER:
-        match = match_transfer
-    else:
-        match = {'$or': [match_create, match_transfer]}
-
-    pipeline = [
-        {'$match': match},
-        {'$unwind': '$block.transactions'},
-        {'$match': match},
-        {'$project': {'block.transactions.id': True}}
-    ]
-    cursor = conn.run(
-        conn.collection('bigchain')
-        .aggregate(pipeline))
-    return (elem['block']['transactions']['id'] for elem in cursor)
-
-
-# TODO: This doesn't seem to be used anywhere
-@register_query(MongoDBConnection)
-def get_asset_by_id(conn, asset_id):
-    cursor = conn.run(
-        conn.collection('bigchain')
-        .aggregate([
-            {'$match': {
-                'block.transactions.id': asset_id,
-                'block.transactions.operation': 'CREATE'
-            }},
-            {'$unwind': '$block.transactions'},
-            {'$match': {
-                'block.transactions.id': asset_id,
-                'block.transactions.operation': 'CREATE'
-            }},
-            {'$project': {'block.transactions.asset': True}}
-        ]))
-    # we need to access some nested fields before returning so lets use a
-    # generator to avoid having to read all records on the cursor at this point
-    return (elem['block']['transactions'] for elem in cursor)
-
-
-@register_query(MongoDBConnection)
-def get_spent(conn, transaction_id, output):
-    cursor = conn.run(
-        conn.collection('bigchain').aggregate([
-            {'$match': {
-                'block.transactions.inputs': {
-                    '$elemMatch': {
-                        'fulfills.transaction_id': transaction_id,
-                        'fulfills.output_index': output,
-                    },
-                },
-            }},
-            {'$unwind': '$block.transactions'},
-            {'$match': {
-                'block.transactions.inputs': {
-                    '$elemMatch': {
-                        'fulfills.transaction_id': transaction_id,
-                        'fulfills.output_index': output,
-                    },
-                },
-            }},
-        ]))
-    # we need to access some nested fields before returning so lets use a
-    # generator to avoid having to read all records on the cursor at this point
-    return (elem['block']['transactions'] for elem in cursor)
-
-
-@register_query(MongoDBConnection)
-def get_spending_transactions(conn, inputs):
-    cursor = conn.run(
-        conn.collection('bigchain').aggregate([
-            {'$match': {
-                'block.transactions.inputs.fulfills': {
-                    '$in': inputs,
-                },
-            }},
-            {'$unwind': '$block.transactions'},
-            {'$match': {
-                'block.transactions.inputs.fulfills': {
-                    '$in': inputs,
-                },
-            }},
-        ]))
-    return ((b['id'], b['block']['transactions']) for b in cursor)
-
-
-@register_query(MongoDBConnection)
-def get_owned_ids(conn, owner):
-    cursor = conn.run(
-        conn.collection('bigchain').aggregate([
-            {'$match': {'block.transactions.outputs.public_keys': owner}},
-            {'$unwind': '$block.transactions'},
-            {'$match': {'block.transactions.outputs.public_keys': owner}}
-        ]))
-    return ((b['id'], b['block']['transactions']) for b in cursor)
-
-
-@register_query(MongoDBConnection)
-def get_votes_by_block_id(conn, block_id):
-    return conn.run(
-        conn.collection('votes')
-        .find({'vote.voting_for_block': block_id},
-              projection={'_id': False}))
-
-
-@register_query(MongoDBConnection)
-def get_votes_for_blocks_by_voter(conn, block_ids, node_pubkey):
-    return conn.run(
-        conn.collection('votes')
-        .find({'vote.voting_for_block': {'$in': block_ids},
-               'node_pubkey': node_pubkey},
-              projection={'_id': False}))
-
-
-@register_query(MongoDBConnection)
-def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey):
-    return conn.run(
-        conn.collection('votes')
-        .find({'vote.voting_for_block': block_id,
-               'node_pubkey': node_pubkey},
-              projection={'_id': False}))
-
-
-@register_query(MongoDBConnection)
-def write_block(conn, block_dict):
-    return conn.run(
-        conn.collection('bigchain')
-        .insert_one(block_dict))
-
-
-@register_query(MongoDBConnection)
-def get_block(conn, block_id):
-    return conn.run(
-        conn.collection('bigchain')
-        .find_one({'id': block_id},
-                  projection={'_id': False}))
-
-
-@register_query(MongoDBConnection)
-def write_assets(conn, assets):
-    try:
-        # unordered means that all the inserts will be attempted instead of
-        # stopping after the first error.
-        return conn.run(
-            conn.collection('assets')
-            .insert_many(assets, ordered=False))
-    # This can happen if we try to write the same asset multiple times.
-    # One case is when we write the same transaction into multiple blocks due
-    # to invalid blocks.
-    # The actual mongodb exception is a BulkWriteError due to a duplicated key
-    # in one of the inserts.
-    except OperationError:
-        return
-
-
-@register_query(MongoDBConnection)
-def get_assets(conn, asset_ids):
-    return conn.run(
-        conn.collection('assets')
-        .find({'id': {'$in': asset_ids}},
-              projection={'_id': False}))
-
-
-@register_query(MongoDBConnection)
-def count_blocks(conn):
-    return conn.run(
-        conn.collection('bigchain')
-        .count())
-
-
-@register_query(MongoDBConnection)
-def count_backlog(conn):
-    return conn.run(
-        conn.collection('backlog')
-        .count())
-
-
-@register_query(MongoDBConnection)
-def write_vote(conn, vote):
-    conn.run(conn.collection('votes').insert_one(vote))
-    vote.pop('_id')
-    return vote
-
-
-@register_query(MongoDBConnection)
-def get_genesis_block(conn):
-    return conn.run(
-        conn.collection('bigchain')
-        .find_one(
-            {'block.transactions.0.operation': 'GENESIS'},
-            {'_id': False}
-        ))
-
-
-@register_query(MongoDBConnection)
-def get_last_voted_block_id(conn, node_pubkey):
-    last_voted = conn.run(
-            conn.collection('votes')
-            .find({'node_pubkey': node_pubkey},
-                  sort=[('vote.timestamp', -1)]))
-
-    # pymongo seems to return a cursor even if there are no results
-    # so we actually need to check the count
-    if last_voted.count() == 0:
-        return get_genesis_block(conn)['id']
-
-    mapping = {v['vote']['previous_block']: v['vote']['voting_for_block']
-               for v in last_voted}
-
-    last_block_id = list(mapping.values())[0]
-
-    explored = set()
-
-    while True:
-        try:
-            if last_block_id in explored:
-                raise CyclicBlockchainError()
-            explored.add(last_block_id)
-            last_block_id = mapping[last_block_id]
-        except KeyError:
-            break
-
-    return last_block_id
-
-
-@register_query(MongoDBConnection)
-def get_new_blocks_feed(conn, start_block_id):
-    namespace = conn.dbname + '.bigchain'
-    match = {'o.id': start_block_id, 'op': 'i', 'ns': namespace}
-    # Neccesary to find in descending order since tests may write same block id several times
-    query = conn.query().local.oplog.rs.find(match).sort('$natural', -1).next()['ts']
-    last_ts = conn.run(query)
-    feed = run_changefeed(conn, 'bigchain', last_ts)
-    return (evt['o'] for evt in feed if evt['op'] == 'i')
-
-
-@register_query(MongoDBConnection)
-def text_search(conn, search, *, language='english', case_sensitive=False,
-                diacritic_sensitive=False, text_score=False, limit=0):
-    cursor = conn.run(
-        conn.collection('assets')
-        .find({'$text': {
-                '$search': search,
-                '$language': language,
-                '$caseSensitive': case_sensitive,
-                '$diacriticSensitive': diacritic_sensitive}},
-              {'score': {'$meta': 'textScore'}, '_id': False})
-        .sort([('score', {'$meta': 'textScore'})])
-        .limit(limit))
-
-    if text_score:
-        return cursor
-
-    return (_remove_text_score(asset) for asset in cursor)
-
-
-def _remove_text_score(asset):
-    asset.pop('score', None)
-    return asset

bigchaindb/backend/mongodb/schema.py

@@ -1,118 +0,0 @@
-"""Utils to initialize and drop the database."""
-
-import logging
-
-from pymongo import ASCENDING, DESCENDING, TEXT
-
-from bigchaindb import backend
-from bigchaindb.common import exceptions
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.mongodb.connection import MongoDBConnection
-
-
-logger = logging.getLogger(__name__)
-register_schema = module_dispatch_registrar(backend.schema)
-
-
-@register_schema(MongoDBConnection)
-def create_database(conn, dbname):
-    if dbname in conn.conn.database_names():
-        raise exceptions.DatabaseAlreadyExists('Database `{}` already exists'
-                                               .format(dbname))
-
-    logger.info('Create database `%s`.', dbname)
-    # TODO: read and write concerns can be declared here
-    conn.conn.get_database(dbname)
-
-
-@register_schema(MongoDBConnection)
-def create_tables(conn, dbname):
-    for table_name in ['bigchain', 'backlog', 'votes', 'assets']:
-        logger.info('Create `%s` table.', table_name)
-        # create the table
-        # TODO: read and write concerns can be declared here
-        conn.conn[dbname].create_collection(table_name)
-
-
-@register_schema(MongoDBConnection)
-def create_indexes(conn, dbname):
-    create_bigchain_secondary_index(conn, dbname)
-    create_backlog_secondary_index(conn, dbname)
-    create_votes_secondary_index(conn, dbname)
-    create_assets_secondary_index(conn, dbname)
-
-
-@register_schema(MongoDBConnection)
-def drop_database(conn, dbname):
-    conn.conn.drop_database(dbname)
-
-
-def create_bigchain_secondary_index(conn, dbname):
-    logger.info('Create `bigchain` secondary index.')
-
-    # to order blocks by timestamp
-    conn.conn[dbname]['bigchain'].create_index([('block.timestamp',
-                                                 ASCENDING)],
-                                               name='block_timestamp')
-
-    # to query the bigchain for a transaction id, this field is unique
-    conn.conn[dbname]['bigchain'].create_index('block.transactions.id',
-                                               name='transaction_id')
-
-    # secondary index for asset uuid, this field is unique
-    conn.conn[dbname]['bigchain']\
-        .create_index('block.transactions.asset.id', name='asset_id')
-
-    # secondary index on the public keys of outputs
-    conn.conn[dbname]['bigchain']\
-        .create_index('block.transactions.outputs.public_keys',
-                      name='outputs')
-
-    # secondary index on inputs/transaction links (transaction_id, output)
-    conn.conn[dbname]['bigchain']\
-        .create_index([
-            ('block.transactions.inputs.fulfills.transaction_id', ASCENDING),
-            ('block.transactions.inputs.fulfills.output_index', ASCENDING),
-        ], name='inputs')
-
-
-def create_backlog_secondary_index(conn, dbname):
-    logger.info('Create `backlog` secondary index.')
-
-    # secondary index on the transaction id with a uniqueness constraint
-    # to make sure there are no duplicated transactions in the backlog
-    conn.conn[dbname]['backlog'].create_index('id',
-                                              name='transaction_id',
-                                              unique=True)
-
-    # compound index to read transactions from the backlog per assignee
-    conn.conn[dbname]['backlog']\
-        .create_index([('assignee', ASCENDING),
-                       ('assignment_timestamp', DESCENDING)],
-                      name='assignee__transaction_timestamp')
-
-
-def create_votes_secondary_index(conn, dbname):
-    logger.info('Create `votes` secondary index.')
-
-    # is the first index redundant then?
-    # compound index to order votes by block id and node
-    conn.conn[dbname]['votes'].create_index([('vote.voting_for_block',
-                                              ASCENDING),
-                                             ('node_pubkey',
-                                              ASCENDING)],
-                                            name='block_and_voter',
-                                            unique=True)
-
-
-def create_assets_secondary_index(conn, dbname):
-    logger.info('Create `assets` secondary index.')
-
-    # unique index on the id of the asset.
-    # the id is the txid of the transaction that created the asset
-    conn.conn[dbname]['assets'].create_index('id',
-                                             name='asset_id',
-                                             unique=True)
-
-    # full text search index
-    conn.conn[dbname]['assets'].create_index([('$**', TEXT)], name='text')

bigchaindb/backend/query.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Query interfaces for backends."""
 
 from functools import singledispatch
@@ -6,11 +11,11 @@ from bigchaindb.backend.exceptions import OperationError
 
 
 @singledispatch
-def write_transaction(connection, signed_transaction):
-    """Write a transaction to the backlog table.
+def store_asset(connection, asset):
+    """Write an asset to the asset table.
 
     Args:
-        signed_transaction (dict): a signed transaction.
+        asset (dict): the asset.
 
     Returns:
         The result of the operation.
@@ -20,104 +25,77 @@ def write_transaction(connection, signed_transaction):
 
 
 @singledispatch
-def update_transaction(connection, transaction_id, doc):
-    """Update a transaction in the backlog table.
+def store_assets(connection, assets):
+    """Write a list of assets to the assets table.
 
     Args:
-        transaction_id (str): the id of the transaction.
-        doc (dict): the values to update.
-
-    Returns:
-        The result of the operation.
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def delete_transaction(connection, *transaction_id):
-    """Delete a transaction from the backlog.
-
-    Args:
-        *transaction_id (str): the transaction(s) to delete.
+        assets (list): a list of assets to write.
 
     Returns:
         The database response.
     """
+
     raise NotImplementedError
 
 
 @singledispatch
-def get_stale_transactions(connection, reassign_delay):
-    """Get a cursor of stale transactions.
-
-    Transactions are considered stale if they have been assigned a node,
-    but are still in the backlog after some amount of time specified in the
-    configuration.
+def store_metadatas(connection, metadata):
+    """Write a list of metadata to metadata table.
 
     Args:
-        reassign_delay (int): threshold (in seconds) to mark a transaction stale.
+        metadata (list): list of metadata.
 
     Returns:
-        A cursor of transactions.
+        The result of the operation.
     """
 
     raise NotImplementedError
 
 
 @singledispatch
-def get_transaction_from_block(connection, transaction_id, block_id):
-    """Get a transaction from a specific block.
-
-    Args:
-        transaction_id (str): the id of the transaction.
-        block_id (str): the id of the block.
-
-    Returns:
-        The matching transaction.
-    """
+def store_transactions(connection, signed_transactions):
+    """Store the list of transactions."""
 
     raise NotImplementedError
 
 
 @singledispatch
-def get_transaction_from_backlog(connection, transaction_id):
-    """Get a transaction from backlog.
+def get_transaction(connection, transaction_id):
+    """Get a transaction from the transactions table.
 
     Args:
         transaction_id (str): the id of the transaction.
 
     Returns:
-        The matching transaction.
+        The result of the operation.
     """
 
     raise NotImplementedError
 
 
 @singledispatch
-def get_blocks_status_from_transaction(connection, transaction_id):
-    """Retrieve block election information given a secondary index and value.
+def get_transactions(connection, transaction_ids):
+    """Get transactions from the transactions table.
 
     Args:
-        value: a value to search (e.g. transaction id string, payload hash string)
-        index (str): name of a secondary index, e.g. 'transaction_id'
+        transaction_ids (list): list of transaction ids to fetch
 
     Returns:
-        :obj:`list` of :obj:`dict`: A list of blocks with with only election information
+        The result of the operation.
     """
 
     raise NotImplementedError
 
 
 @singledispatch
-def get_asset_by_id(conneciton, asset_id):
-    """Returns the asset associated with an asset_id.
+def get_asset(connection, asset_id):
+    """Get a transaction from the transactions table.
 
     Args:
-        asset_id (str): The asset id.
+        asset_id (str): the id of the asset
 
     Returns:
-        Returns a rethinkdb cursor.
+        The result of the operation.
     """
 
     raise NotImplementedError
@@ -170,63 +148,6 @@ def get_owned_ids(connection, owner):
     raise NotImplementedError
 
 
-@singledispatch
-def get_votes_by_block_id(connection, block_id):
-    """Get all the votes casted for a specific block.
-
-    Args:
-        block_id (str): the block id to use.
-
-    Returns:
-        A cursor for the matching votes.
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def get_votes_by_block_id_and_voter(connection, block_id, node_pubkey):
-    """Get all the votes casted for a specific block by a specific voter.
-
-    Args:
-        block_id (str): the block id to use.
-        node_pubkey (str): base58 encoded public key
-
-    Returns:
-        A cursor for the matching votes.
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def get_votes_for_blocks_by_voter(connection, block_ids, pubkey):
-    """Return votes for many block_ids
-
-    Args:
-        block_ids (set): block_ids
-        pubkey (str): public key of voting node
-
-    Returns:
-        A cursor of votes matching given block_ids and public key
-    """
-    raise NotImplementedError
-
-
-@singledispatch
-def write_block(connection, block):
-    """Write a block to the bigchain table.
-
-    Args:
-        block (dict): the block to write.
-
-    Returns:
-        The database response.
-    """
-
-    raise NotImplementedError
-
-
 @singledispatch
 def get_block(connection, block_id):
     """Get a block from the bigchain table.
@@ -242,14 +163,29 @@ def get_block(connection, block_id):
 
 
 @singledispatch
-def write_assets(connection, assets):
-    """Write a list of assets to the assets table.
+def get_block_with_transaction(connection, txid):
+    """Get a block containing transaction id `txid`
 
     Args:
-        assets (list): a list of assets to write.
+        txid (str): id of transaction to be searched.
 
     Returns:
-        The database response.
+        block_id (int): the block id or `None`
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_metadata(connection, transaction_ids):
+    """Get a list of metadata from the metadata table.
+
+    Args:
+        transaction_ids (list): a list of ids for the metadata to be retrieved from
+        the database.
+
+    Returns:
+        metadata (list): the list of returned metadata.
     """
     raise NotImplementedError
 
@@ -257,83 +193,18 @@ def write_assets(connection, assets):
 @singledispatch
 def get_assets(connection, asset_ids):
     """Get a list of assets from the assets table.
-
     Args:
         asset_ids (list): a list of ids for the assets to be retrieved from
         the database.
-
     Returns:
         assets (list): the list of returned assets.
     """
     raise NotImplementedError
 
 
-@singledispatch
-def count_blocks(connection):
-    """Count the number of blocks in the bigchain table.
-
-    Returns:
-        The number of blocks.
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def count_backlog(connection):
-    """Count the number of transactions in the backlog table.
-
-    Returns:
-        The number of transactions in the backlog.
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def write_vote(connection, vote):
-    """Write a vote to the votes table.
-
-    Args:
-        vote (dict): the vote to write.
-
-    Returns:
-        The database response.
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def get_genesis_block(connection):
-    """Get the genesis block.
-
-    Returns:
-        The genesis block
-    """
-
-    raise NotImplementedError
-
-
-@singledispatch
-def get_last_voted_block_id(connection, node_pubkey):
-    """Get the last voted block for a specific node.
-
-    Args:
-        node_pubkey (str): base58 encoded public key.
-
-    Returns:
-        The id of the last block the node has voted on. If the node didn't cast
-        any vote then the genesis block id is returned.
-    """
-
-    raise NotImplementedError
-
-
 @singledispatch
 def get_txids_filtered(connection, asset_id, operation=None):
-    """
-    Return all transactions for a particular asset id and optional operation.
+    """Return all transactions for a particular asset id and optional operation.
 
     Args:
         asset_id (str): ID of transaction that defined the asset
@@ -343,24 +214,9 @@ def get_txids_filtered(connection, asset_id, operation=None):
     raise NotImplementedError
 
 
-@singledispatch
-def get_new_blocks_feed(connection, start_block_id):
-    """
-    Return a generator that yields change events of the blocks feed
-
-    Args:
-        start_block_id (str): ID of block to resume from
-
-    Returns:
-        Generator of change events
-    """
-
-    raise NotImplementedError
-
-
 @singledispatch
 def text_search(conn, search, *, language='english', case_sensitive=False,
-                diacritic_sensitive=False, text_score=False, limit=0):
+                diacritic_sensitive=False, text_score=False, limit=0, table=None):
     """Return all the assets that match the text search.
 
     The results are sorted by text score.
@@ -389,3 +245,186 @@ def text_search(conn, search, *, language='english', case_sensitive=False,
 
     raise OperationError('This query is only supported when running '
                          'BigchainDB with MongoDB as the backend.')
+
+
+@singledispatch
+def get_latest_block(conn):
+    """Get the latest commited block i.e. block with largest height"""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_block(conn, block):
+    """Write a new block to the `blocks` table
+
+    Args:
+        block (dict): block with current height and block hash.
+
+    Returns:
+        The result of the operation.
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_unspent_outputs(connection, unspent_outputs):
+    """Store unspent outputs in ``utxo_set`` table."""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def delete_unspent_outputs(connection, unspent_outputs):
+    """Delete unspent outputs in ``utxo_set`` table."""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def delete_transactions(conn, txn_ids):
+    """Delete transactions from database
+
+    Args:
+        txn_ids (list): list of transaction ids
+
+    Returns:
+        The result of the operation.
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_unspent_outputs(connection, *, query=None):
+    """Retrieves unspent outputs.
+
+    Args:
+        query (dict): An optional parameter to filter the result set.
+            Defaults to ``None``, which means that all UTXO records
+            will be returned.
+
+    Returns:
+        Generator yielding unspent outputs (UTXO set) according to the
+        given query.
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_pre_commit_state(connection, state):
+    """Store pre-commit state.
+
+    Args:
+        state (dict): pre-commit state.
+
+    Returns:
+        The result of the operation.
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_pre_commit_state(connection):
+    """Get pre-commit state.
+
+    Returns:
+        Document representing the pre-commit state.
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_validator_set(conn, validator_update):
+    """Store updated validator set"""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def delete_validator_set(conn, height):
+    """Delete the validator set at the given height."""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_election(conn, election_id, height, is_concluded):
+    """Store election record"""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_elections(conn, elections):
+    """Store election records in bulk"""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def delete_elections(conn, height):
+    """Delete all election records at the given height"""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_validator_set(conn, height):
+    """Get validator set for a given `height`, if `height` is not specified
+    then return the latest validator set
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_election(conn, election_id):
+    """Return the election record
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_asset_tokens_for_public_key(connection, asset_id, public_key):
+    """Retrieve a list of tokens of type `asset_id` that are owned by the `public_key`.
+    Args:
+        asset_id (str): Id of the token.
+        public_key (str): base58 encoded public key
+    Returns:
+        Iterator of transaction that list given owner in conditions.
+    """
+    raise NotImplementedError
+
+
+@singledispatch
+def store_abci_chain(conn, height, chain_id, is_synced=True):
+    """Create or update an ABCI chain at the given height.
+    Usually invoked in the beginning of the ABCI communications (height=0)
+    or when ABCI client (like Tendermint) is migrated (any height).
+
+    Args:
+        is_synced: True if the chain is known by both ABCI client and server
+    """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def delete_abci_chain(conn, height):
+    """Delete the ABCI chain at the given height."""
+
+    raise NotImplementedError
+
+
+@singledispatch
+def get_latest_abci_chain(conn):
+    """Returns the ABCI chain stored at the biggest height, if any,
+    None otherwise.
+    """
+    raise NotImplementedError

bigchaindb/backend/rethinkdb/__init__.py

@@ -1,22 +0,0 @@
-"""RethinkDB backend implementation.
-
-Contains a RethinkDB-specific implementation of the
-:mod:`~bigchaindb.backend.changefeed`, :mod:`~bigchaindb.backend.query`, and
-:mod:`~bigchaindb.backend.schema` interfaces.
-
-You can specify BigchainDB to use RethinkDB as its database backend by either
-setting ``database.backend`` to ``'rethinkdb'`` in your configuration file, or
-setting the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable to
-``'rethinkdb'``.
-
-If configured to use RethinkDB, BigchainDB will automatically return instances
-of :class:`~bigchaindb.backend.rethinkdb.RethinkDBConnection` for
-:func:`~bigchaindb.backend.connection.connect` and dispatch calls of the
-generic backend interfaces to the implementations in this module.
-"""
-
-# Register the single dispatched modules on import.
-from bigchaindb.backend.rethinkdb import admin, changefeed, schema, query  # noqa
-
-# RethinkDBConnection should always be accessed via
-# ``bigchaindb.backend.connect()``.

bigchaindb/backend/rethinkdb/admin.py

@@ -1,165 +0,0 @@
-"""Database configuration functions."""
-import logging
-
-import rethinkdb as r
-
-from bigchaindb.backend import admin
-from bigchaindb.backend.schema import TABLES
-from bigchaindb.backend.exceptions import OperationError
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection
-
-logger = logging.getLogger(__name__)
-
-register_admin = module_dispatch_registrar(admin)
-
-
-@register_admin(RethinkDBConnection)
-def get_config(connection, *, table):
-    """Get the configuration of the given table.
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        table (str): The name of the table to get the configuration for.
-
-    Returns:
-        dict: The configuration of the given table
-
-    """
-    return connection.run(r.table(table).config())
-
-
-@register_admin(RethinkDBConnection)
-def reconfigure(connection, *, table, shards, replicas,
-                primary_replica_tag=None, dry_run=False,
-                nonvoting_replica_tags=None):
-    """Reconfigures the given table.
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        table (str): The name of the table to reconfigure.
-        shards (int): The number of shards, an integer from 1-64.
-        replicas (:obj:`int` | :obj:`dict`):
-            * If replicas is an integer, it specifies the number of
-              replicas per shard. Specifying more replicas than there
-              are servers will return an error.
-            * If replicas is a dictionary, it specifies key-value pairs
-              of server tags and the number of replicas to assign to
-              those servers::
-
-                  {'africa': 2, 'asia': 4, 'europe': 2, ...}
-        primary_replica_tag (str): The primary server specified by its
-            server tag. Required if ``replicas`` is a dictionary. The
-            tag must be in the ``replicas`` dictionary. This must not be
-            specified if ``replicas`` is an integer. Defaults to
-            ``None``.
-        dry_run (bool): If ``True`` the generated configuration will not
-            be applied to the table, only returned. Defaults to
-            ``False``.
-        nonvoting_replica_tags (:obj:`list` of :obj:`str`): Replicas
-            with these server tags will be added to the
-            ``nonvoting_replicas`` list of the resulting configuration.
-            Defaults to ``None``.
-
-    Returns:
-        dict: A dictionary with possibly three keys:
-
-            * ``reconfigured``: the number of tables reconfigured. This
-              will be ``0`` if ``dry_run`` is ``True``.
-            * ``config_changes``: a list of new and old table
-              configuration values.
-            * ``status_changes``: a list of new and old table status
-              values.
-
-            For more information please consult RethinkDB's
-            documentation `ReQL command: reconfigure
-            <https://rethinkdb.com/api/python/reconfigure/>`_.
-
-    Raises:
-        OperationError: If the reconfiguration fails due to a
-            RethinkDB :exc:`ReqlOpFailedError` or
-            :exc:`ReqlQueryLogicError`.
-
-    """
-    params = {
-        'shards': shards,
-        'replicas': replicas,
-        'dry_run': dry_run,
-    }
-    if primary_replica_tag:
-        params.update(
-            primary_replica_tag=primary_replica_tag,
-            nonvoting_replica_tags=nonvoting_replica_tags,
-        )
-    try:
-        return connection.run(r.table(table).reconfigure(**params))
-    except (r.ReqlOpFailedError, r.ReqlQueryLogicError) as e:
-        raise OperationError('Failed to reconfigure tables.') from e
-
-
-@register_admin(RethinkDBConnection)
-def set_shards(connection, *, shards, dry_run=False):
-    """Sets the shards for the tables
-    :const:`~bigchaindb.backend.schema.TABLES`.
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        shards (int): The number of shards, an integer from 1-64.
-        dry_run (bool): If ``True`` the generated configuration will not
-            be applied to the table, only returned. Defaults to
-            ``False``.
-
-    Returns:
-        dict: A dictionary with the configuration and status changes.
-            For more details please see :func:`.reconfigure`.
-
-    """
-    changes = {}
-    for table in TABLES:
-        replicas = len(
-            get_config(connection, table=table)['shards'][0]['replicas'])
-        change = reconfigure(
-            connection,
-            table=table,
-            shards=shards,
-            replicas=replicas,
-            dry_run=dry_run,
-        )
-        changes[table] = change
-    return changes
-
-
-@register_admin(RethinkDBConnection)
-def set_replicas(connection, *, replicas, dry_run=False):
-    """Sets the replicas for the tables
-    :const:`~bigchaindb.backend.schema.TABLES`.
-
-    Args:
-        connection (:class:`~bigchaindb.backend.connection.Connection`):
-            A connection to the database.
-        replicas (int): The number of replicas per shard. Specifying
-            more replicas than there are servers will return an error.
-        dry_run (bool): If ``True`` the generated configuration will not
-            be applied to the table, only returned. Defaults to
-            ``False``.
-
-    Returns:
-        dict: A dictionary with the configuration and status changes.
-            For more details please see :func:`.reconfigure`.
-
-    """
-    changes = {}
-    for table in TABLES:
-        shards = len(get_config(connection, table=table)['shards'])
-        change = reconfigure(
-            connection,
-            table=table,
-            shards=shards,
-            replicas=replicas,
-            dry_run=dry_run,
-        )
-        changes[table] = change
-    return changes

bigchaindb/backend/rethinkdb/changefeed.py

@@ -1,59 +0,0 @@
-import time
-import logging
-import rethinkdb as r
-
-from bigchaindb import backend
-from bigchaindb.backend.exceptions import BackendError
-from bigchaindb.backend.changefeed import ChangeFeed
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection
-
-
-logger = logging.getLogger(__name__)
-register_changefeed = module_dispatch_registrar(backend.changefeed)
-
-
-class RethinkDBChangeFeed(ChangeFeed):
-    """This class wraps a RethinkDB changefeed as a multipipes Node."""
-
-    def run_forever(self):
-        for element in self.prefeed:
-            self.outqueue.put(element)
-
-        for change in run_changefeed(self.connection, self.table):
-            is_insert = change['old_val'] is None
-            is_delete = change['new_val'] is None
-            is_update = not is_insert and not is_delete
-
-            if is_insert and (self.operation & ChangeFeed.INSERT):
-                self.outqueue.put(change['new_val'])
-            elif is_delete and (self.operation & ChangeFeed.DELETE):
-                self.outqueue.put(change['old_val'])
-            elif is_update and (self.operation & ChangeFeed.UPDATE):
-                self.outqueue.put(change['new_val'])
-
-
-def run_changefeed(connection, table):
-    """Encapsulate operational logic of tailing changefeed from RethinkDB
-    """
-    while True:
-        try:
-            for change in connection.run(r.table(table).changes()):
-                yield change
-            break
-        except (BackendError, r.ReqlDriverError) as exc:
-            logger.exception('Error connecting to the database, retrying')
-            time.sleep(1)
-
-
-@register_changefeed(RethinkDBConnection)
-def get_changefeed(connection, table, operation, *, prefeed=None):
-    """Return a RethinkDB changefeed.
-
-    Returns:
-        An instance of
-        :class:`~bigchaindb.backend.rethinkdb.RethinkDBChangeFeed`.
-    """
-
-    return RethinkDBChangeFeed(table, operation, prefeed=prefeed,
-                               connection=connection)

bigchaindb/backend/rethinkdb/connection.py

@@ -1,47 +0,0 @@
-import rethinkdb as r
-from bigchaindb.backend.connection import Connection
-from bigchaindb.backend.exceptions import ConnectionError, OperationError
-
-
-class RethinkDBConnection(Connection):
-    """
-    This class is a proxy to run queries against the database, it is:
-
-        - lazy, since it creates a connection only when needed
-        - resilient, because before raising exceptions it tries
-          more times to run the query or open a connection.
-    """
-
-    def run(self, query):
-        """Run a RethinkDB query.
-
-        Args:
-            query: the RethinkDB query.
-
-        Raises:
-            :exc:`rethinkdb.ReqlDriverError`: After
-                :attr:`~.RethinkDBConnection.max_tries`.
-        """
-
-        try:
-            return query.run(self.conn)
-        except r.ReqlDriverError as exc:
-            raise OperationError from exc
-
-    def _connect(self):
-        """Set a connection to RethinkDB.
-
-        The connection is available via :attr:`~.RethinkDBConnection.conn`.
-
-        Raises:
-            :exc:`rethinkdb.ReqlDriverError`: After
-                :attr:`~.RethinkDBConnection.max_tries`.
-        """
-
-        try:
-            return r.connect(host=self.host,
-                             port=self.port,
-                             db=self.dbname,
-                             timeout=self.connection_timeout)
-        except (r.ReqlDriverError, r.ReqlTimeoutError) as exc:
-            raise ConnectionError from exc

bigchaindb/backend/rethinkdb/query.py

@@ -1,298 +0,0 @@
-from itertools import chain
-import logging as logger
-from time import time
-
-import rethinkdb as r
-
-from bigchaindb import backend, utils
-from bigchaindb.backend.rethinkdb import changefeed
-from bigchaindb.common import exceptions
-from bigchaindb.common.transaction import Transaction
-from bigchaindb.common.utils import serialize
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection
-
-
-logger = logger.getLogger(__name__)
-
-
-READ_MODE = 'majority'
-WRITE_DURABILITY = 'hard'
-
-register_query = module_dispatch_registrar(backend.query)
-
-
-@register_query(RethinkDBConnection)
-def write_transaction(connection, signed_transaction):
-    return connection.run(
-            r.table('backlog')
-            .insert(signed_transaction, durability=WRITE_DURABILITY))
-
-
-@register_query(RethinkDBConnection)
-def update_transaction(connection, transaction_id, doc):
-    return connection.run(
-            r.table('backlog')
-            .get(transaction_id)
-            .update(doc))
-
-
-@register_query(RethinkDBConnection)
-def delete_transaction(connection, *transaction_id):
-    return connection.run(
-            r.table('backlog')
-            .get_all(*transaction_id)
-            .delete(durability=WRITE_DURABILITY))
-
-
-@register_query(RethinkDBConnection)
-def get_stale_transactions(connection, reassign_delay):
-    return connection.run(
-            r.table('backlog')
-            .filter(lambda tx: time() - tx['assignment_timestamp'] > reassign_delay))
-
-
-@register_query(RethinkDBConnection)
-def get_transaction_from_block(connection, transaction_id, block_id):
-    return connection.run(
-            r.table('bigchain', read_mode=READ_MODE)
-            .get(block_id)
-            .get_field('block')
-            .get_field('transactions')
-            .filter(lambda tx: tx['id'] == transaction_id))[0]
-
-
-@register_query(RethinkDBConnection)
-def get_transaction_from_backlog(connection, transaction_id):
-    return connection.run(
-            r.table('backlog')
-            .get(transaction_id)
-            .without('assignee', 'assignment_timestamp')
-            .default(None))
-
-
-@register_query(RethinkDBConnection)
-def get_blocks_status_from_transaction(connection, transaction_id):
-    return connection.run(
-            r.table('bigchain', read_mode=READ_MODE)
-            .get_all(transaction_id, index='transaction_id')
-            .pluck('votes', 'id', {'block': ['voters']}))
-
-
-@register_query(RethinkDBConnection)
-def get_txids_filtered(connection, asset_id, operation=None):
-    # here we only want to return the transaction ids since later on when
-    # we are going to retrieve the transaction with status validation
-
-    parts = []
-
-    if operation in (Transaction.CREATE, None):
-        # First find the asset's CREATE transaction
-        parts.append(connection.run(
-            _get_asset_create_tx_query(asset_id).get_field('id')))
-
-    if operation in (Transaction.TRANSFER, None):
-        # Then find any TRANSFER transactions related to the asset
-        parts.append(connection.run(
-            r.table('bigchain')
-            .get_all(asset_id, index='asset_id')
-            .concat_map(lambda block: block['block']['transactions'])
-            .filter(lambda transaction: transaction['asset']['id'] == asset_id)
-            .get_field('id')))
-
-    return chain(*parts)
-
-
-@register_query(RethinkDBConnection)
-def get_asset_by_id(connection, asset_id):
-    return connection.run(_get_asset_create_tx_query(asset_id).pluck('asset'))
-
-
-def _get_asset_create_tx_query(asset_id):
-    return r.table('bigchain', read_mode=READ_MODE) \
-            .get_all(asset_id, index='transaction_id') \
-            .concat_map(lambda block: block['block']['transactions']) \
-            .filter(lambda transaction: transaction['id'] == asset_id)
-
-
-@register_query(RethinkDBConnection)
-def get_spent(connection, transaction_id, output):
-    return connection.run(
-            r.table('bigchain', read_mode=READ_MODE)
-             .get_all([transaction_id, output], index='inputs')
-             .concat_map(lambda doc: doc['block']['transactions'])
-             .filter(lambda transaction: transaction['inputs'].contains(
-                lambda input_: input_['fulfills'] == {
-                    'transaction_id': transaction_id, 'output_index': output})))
-
-
-@register_query(RethinkDBConnection)
-def get_owned_ids(connection, owner):
-    query = (r.table('bigchain', read_mode=READ_MODE)
-             .get_all(owner, index='outputs')
-             .distinct()
-             .concat_map(unwind_block_transactions)
-             .filter(lambda doc: doc['tx']['outputs'].contains(
-                lambda c: c['public_keys'].contains(owner))))
-    cursor = connection.run(query)
-    return ((b['id'], b['tx']) for b in cursor)
-
-
-@register_query(RethinkDBConnection)
-def get_votes_by_block_id(connection, block_id):
-    return connection.run(
-            r.table('votes', read_mode=READ_MODE)
-            .between([block_id, r.minval], [block_id, r.maxval], index='block_and_voter')
-            .without('id'))
-
-
-@register_query(RethinkDBConnection)
-def get_votes_by_block_id_and_voter(connection, block_id, node_pubkey):
-    return connection.run(
-            r.table('votes')
-            .get_all([block_id, node_pubkey], index='block_and_voter')
-            .without('id'))
-
-
-@register_query(RethinkDBConnection)
-def write_block(connection, block_dict):
-    return connection.run(
-            r.table('bigchain')
-            .insert(r.json(serialize(block_dict)), durability=WRITE_DURABILITY))
-
-
-@register_query(RethinkDBConnection)
-def get_block(connection, block_id):
-    return connection.run(r.table('bigchain').get(block_id))
-
-
-@register_query(RethinkDBConnection)
-def write_assets(connection, assets):
-    return connection.run(
-            r.table('assets')
-            .insert(assets, durability=WRITE_DURABILITY))
-
-
-@register_query(RethinkDBConnection)
-def get_assets(connection, asset_ids):
-    return connection.run(
-            r.table('assets', read_mode=READ_MODE)
-            .get_all(*asset_ids))
-
-
-@register_query(RethinkDBConnection)
-def count_blocks(connection):
-    return connection.run(
-            r.table('bigchain', read_mode=READ_MODE)
-            .count())
-
-
-@register_query(RethinkDBConnection)
-def count_backlog(connection):
-    return connection.run(
-            r.table('backlog', read_mode=READ_MODE)
-            .count())
-
-
-@register_query(RethinkDBConnection)
-def write_vote(connection, vote):
-    return connection.run(
-            r.table('votes')
-            .insert(vote))
-
-
-@register_query(RethinkDBConnection)
-def get_genesis_block(connection):
-    return connection.run(
-        r.table('bigchain', read_mode=READ_MODE)
-        .filter(utils.is_genesis_block)
-        .nth(0))
-
-
-@register_query(RethinkDBConnection)
-def get_last_voted_block_id(connection, node_pubkey):
-    try:
-        # get the latest value for the vote timestamp (over all votes)
-        max_timestamp = connection.run(
-            r.table('votes', read_mode=READ_MODE)
-            .filter(r.row['node_pubkey'] == node_pubkey)
-            .max(r.row['vote']['timestamp']))['vote']['timestamp']
-
-        last_voted = list(connection.run(
-            r.table('votes', read_mode=READ_MODE)
-            .filter(r.row['vote']['timestamp'] == max_timestamp)
-            .filter(r.row['node_pubkey'] == node_pubkey)))
-
-    except r.ReqlNonExistenceError:
-        # return last vote if last vote exists else return Genesis block
-        return get_genesis_block(connection)['id']
-
-    # Now the fun starts. Since the resolution of timestamp is a second,
-    # we might have more than one vote per timestamp. If this is the case
-    # then we need to rebuild the chain for the blocks that have been retrieved
-    # to get the last one.
-
-    # Given a block_id, mapping returns the id of the block pointing at it.
-    mapping = {v['vote']['previous_block']: v['vote']['voting_for_block']
-               for v in last_voted}
-
-    # Since we follow the chain backwards, we can start from a random
-    # point of the chain and "move up" from it.
-    last_block_id = list(mapping.values())[0]
-
-    # We must be sure to break the infinite loop. This happens when:
-    # - the block we are currenty iterating is the one we are looking for.
-    #   This will trigger a KeyError, breaking the loop
-    # - we are visiting again a node we already explored, hence there is
-    #   a loop. This might happen if a vote points both `previous_block`
-    #   and `voting_for_block` to the same `block_id`
-    explored = set()
-
-    while True:
-        try:
-            if last_block_id in explored:
-                raise exceptions.CyclicBlockchainError()
-            explored.add(last_block_id)
-            last_block_id = mapping[last_block_id]
-        except KeyError:
-            break
-
-    return last_block_id
-
-
-@register_query(RethinkDBConnection)
-def get_new_blocks_feed(connection, start_block_id):  # pragma: no cover
-    logger.warning('RethinkDB changefeed unable to resume from given block: %s',
-                   start_block_id)
-    # In order to get blocks in the correct order, it may be acceptable to
-    # look in the votes table to see what order other nodes have used.
-    for change in changefeed.run_changefeed(connection, 'bigchain'):
-        yield change['new_val']
-
-
-@register_query(RethinkDBConnection)
-def get_votes_for_blocks_by_voter(connection, block_ids, node_pubkey):
-    return connection.run(
-        r.table('votes')
-        .filter(lambda row: r.expr(block_ids).contains(row['vote']['voting_for_block']))
-        .filter(lambda row: row['node_pubkey'] == node_pubkey))
-
-
-def unwind_block_transactions(block):
-    """ Yield a block for each transaction in given block """
-    return block['block']['transactions'].map(lambda tx: block.merge({'tx': tx}))
-
-
-@register_query(RethinkDBConnection)
-def get_spending_transactions(connection, links):
-    query = (
-        r.table('bigchain')
-        .get_all(*[(l['transaction_id'], l['output_index']) for l in links],
-                 index='inputs')
-        .concat_map(unwind_block_transactions)
-        # filter transactions spending output
-        .filter(lambda doc: r.expr(links).set_intersection(
-            doc['tx']['inputs'].map(lambda i: i['fulfills'])))
-    )
-    cursor = connection.run(query)
-    return ((b['id'], b['tx']) for b in cursor)

bigchaindb/backend/rethinkdb/schema.py

@@ -1,130 +0,0 @@
-import logging
-
-import rethinkdb as r
-
-from bigchaindb import backend
-from bigchaindb.common import exceptions
-from bigchaindb.backend.utils import module_dispatch_registrar
-from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection
-
-
-logger = logging.getLogger(__name__)
-register_schema = module_dispatch_registrar(backend.schema)
-
-
-@register_schema(RethinkDBConnection)
-def create_database(connection, dbname):
-    if connection.run(r.db_list().contains(dbname)):
-        raise exceptions.DatabaseAlreadyExists('Database `{}` already exists'.format(dbname))
-
-    logger.info('Create database `%s`.', dbname)
-    connection.run(r.db_create(dbname))
-
-
-@register_schema(RethinkDBConnection)
-def create_tables(connection, dbname):
-    for table_name in ['bigchain', 'backlog', 'votes', 'assets']:
-        logger.info('Create `%s` table.', table_name)
-        connection.run(r.db(dbname).table_create(table_name))
-
-
-@register_schema(RethinkDBConnection)
-def create_indexes(connection, dbname):
-    create_bigchain_secondary_index(connection, dbname)
-    create_backlog_secondary_index(connection, dbname)
-    create_votes_secondary_index(connection, dbname)
-
-
-@register_schema(RethinkDBConnection)
-def drop_database(connection, dbname):
-    try:
-        logger.info('Drop database `%s`', dbname)
-        connection.run(r.db_drop(dbname))
-        logger.info('Done.')
-    except r.ReqlOpFailedError:
-        raise exceptions.DatabaseDoesNotExist('Database `{}` does not exist'.format(dbname))
-
-
-def create_bigchain_secondary_index(connection, dbname):
-    logger.info('Create `bigchain` secondary index.')
-
-    # to order blocks by timestamp
-    connection.run(
-        r.db(dbname)
-        .table('bigchain')
-        .index_create('block_timestamp', r.row['block']['timestamp']))
-
-    # to query the bigchain for a transaction id
-    connection.run(
-        r.db(dbname)
-        .table('bigchain')
-        .index_create('transaction_id', r.row['block']['transactions']['id'], multi=True))
-
-    # secondary index for asset links (in TRANSFER transactions)
-    connection.run(
-        r.db(dbname)
-        .table('bigchain')
-        .index_create('asset_id', r.row['block']['transactions']['asset']['id'], multi=True))
-
-    # secondary index on the public keys of outputs
-    # the last reduce operation is to return a flatten list of public_keys
-    # without it we would need to match exactly the public_keys list.
-    # For instance querying for `pk1` would not match documents with
-    # `public_keys: [pk1, pk2, pk3]`
-    connection.run(
-        r.db(dbname)
-         .table('bigchain')
-         .index_create('outputs',
-                       r.row['block']['transactions']
-                        .concat_map(lambda tx: tx['outputs']['public_keys'])
-                        .reduce(lambda l, r: l + r), multi=True))
-
-    # secondary index on inputs/transaction links (transaction_id, output)
-    connection.run(
-        r.db(dbname)
-         .table('bigchain')
-         .index_create('inputs',
-                       r.row['block']['transactions']
-                        .concat_map(lambda tx: tx['inputs']['fulfills'])
-                        .with_fields('transaction_id', 'output_index')
-                        .map(lambda fulfills: [fulfills['transaction_id'],
-                                               fulfills['output_index']]),
-                       multi=True))
-
-    # wait for rethinkdb to finish creating secondary indexes
-    connection.run(
-        r.db(dbname)
-        .table('bigchain')
-        .index_wait())
-
-
-def create_backlog_secondary_index(connection, dbname):
-    logger.info('Create `backlog` secondary index.')
-
-    # compound index to read transactions from the backlog per assignee
-    connection.run(
-        r.db(dbname)
-        .table('backlog')
-        .index_create('assignee__transaction_timestamp', [r.row['assignee'], r.row['assignment_timestamp']]))
-
-    # wait for rethinkdb to finish creating secondary indexes
-    connection.run(
-        r.db(dbname)
-        .table('backlog')
-        .index_wait())
-
-
-def create_votes_secondary_index(connection, dbname):
-    logger.info('Create `votes` secondary index.')
-
-    # compound index to order votes by block id and node
-    connection.run(
-        r.db(dbname)
-        .table('votes')
-        .index_create('block_and_voter', [r.row['vote']['voting_for_block'], r.row['node_pubkey']]))
-
-    # wait for rethinkdb to finish creating secondary indexes
-    connection.run(
-        r.db(dbname)
-        .table('votes')
-        .index_wait())

bigchaindb/backend/schema.py

@@ -1,25 +1,29 @@
-"""Database creation and schema-providing interfaces for backends.
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
 
-Attributes:
-    TABLES (tuple): The three standard tables BigchainDB relies on:
-
-        * ``backlog`` for incoming transactions awaiting to be put into
-          a block.
-        * ``bigchain`` for blocks.
-        * ``votes`` to store votes for each block by each federation
-          node.
-
-"""
+"""Database creation and schema-providing interfaces for backends."""
 
 from functools import singledispatch
 import logging
 
 import bigchaindb
 from bigchaindb.backend.connection import connect
+from bigchaindb.common.exceptions import ValidationError
+from bigchaindb.common.utils import validate_all_values_for_key_in_obj, validate_all_values_for_key_in_list
 
 logger = logging.getLogger(__name__)
 
-TABLES = ('bigchain', 'backlog', 'votes')
+# Tables/collections that every backend database must create
+TABLES = ('transactions', 'blocks', 'assets', 'metadata',
+          'validators', 'elections', 'pre_commit', 'utxos', 'abci_chains')
+
+VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german',
+                   'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian',
+                   'russian', 'spanish', 'swedish', 'turkish', 'none',
+                   'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt',
+                   'ro', 'ru', 'es', 'sv', 'tr')
 
 
 @singledispatch
@@ -28,10 +32,6 @@ def create_database(connection, dbname):
 
     Args:
         dbname (str): the name of the database to create.
-
-    Raises:
-        :exc:`~DatabaseAlreadyExists`: If the given :attr:`dbname` already
-            exists as a database.
     """
 
     raise NotImplementedError
@@ -48,17 +48,6 @@ def create_tables(connection, dbname):
     raise NotImplementedError
 
 
-@singledispatch
-def create_indexes(connection, dbname):
-    """Create the indexes to be used by BigchainDB.
-
-    Args:
-        dbname (str): the name of the database to create indexes for.
-    """
-
-    raise NotImplementedError
-
-
 @singledispatch
 def drop_database(connection, dbname):
     """Drop the database used by BigchainDB.
@@ -87,10 +76,6 @@ def init_database(connection=None, dbname=None):
         dbname (str): the name of the database to create.
             Defaults to the database name given in the BigchainDB
             configuration.
-
-    Raises:
-        :exc:`~DatabaseAlreadyExists`: If the given :attr:`dbname` already
-            exists as a database.
     """
 
     connection = connection or connect()
@@ -98,4 +83,46 @@ def init_database(connection=None, dbname=None):
 
     create_database(connection, dbname)
     create_tables(connection, dbname)
-    create_indexes(connection, dbname)
+
+
+def validate_language_key(obj, key):
+    """Validate all nested "language" key in `obj`.
+
+       Args:
+           obj (dict): dictionary whose "language" key is to be validated.
+
+       Returns:
+           None: validation successful
+
+        Raises:
+            ValidationError: will raise exception in case language is not valid.
+    """
+    backend = bigchaindb.config['database']['backend']
+
+    if backend == 'localmongodb':
+        data = obj.get(key, {})
+        if isinstance(data, dict):
+            validate_all_values_for_key_in_obj(data, 'language', validate_language)
+        elif isinstance(data, list):
+            validate_all_values_for_key_in_list(data, 'language', validate_language)
+
+
+def validate_language(value):
+    """Check if `value` is a valid language.
+       https://docs.mongodb.com/manual/reference/text-search-languages/
+
+        Args:
+            value (str): language to validated
+
+        Returns:
+            None: validation successful
+
+        Raises:
+            ValidationError: will raise exception in case language is not valid.
+    """
+    if value not in VALID_LANGUAGES:
+        error_str = ('MongoDB does not support text search for the '
+                     'language "{}". If you do not understand this error '
+                     'message then please rename key/field "language" to '
+                     'something else like "lang".').format(value)
+        raise ValidationError(error_str)

bigchaindb/backend/utils.py

@@ -1,6 +1,15 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+import bigchaindb
+
+
 class ModuleDispatchRegistrationError(Exception):
     """Raised when there is a problem registering dispatched functions for a
-    module"""
+    module
+    """
 
 
 def module_dispatch_registrar(module):
@@ -15,6 +24,16 @@ def module_dispatch_registrar(module):
                     ('`{module}` does not contain a single-dispatchable '
                      'function named `{func}`. The module being registered '
                      'was not implemented correctly!').format(
-                         func=func_name, module=module.__name__)) from ex
+                        func=func_name, module=module.__name__)) from ex
+
         return wrapper
+
     return dispatch_wrapper
+
+
+def get_bigchaindb_config_value(key, default_value=None):
+    return bigchaindb.config['database'].get(key, default_value)
+
+
+def get_bigchaindb_config_value_or_key_error(key):
+    return bigchaindb.config['database'][key]

bigchaindb/commands/bigchaindb.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Implementation of the `bigchaindb` command,
 the command-line interface (CLI) for BigchainDB Server.
 """
@@ -9,25 +14,24 @@ import copy
 import json
 import sys
 
-from bigchaindb.common import crypto
-from bigchaindb.common.exceptions import (StartupError,
-                                          DatabaseAlreadyExists,
-                                          KeypairNotFoundException,
-                                          DatabaseDoesNotExist)
+from bigchaindb.core import rollback
+from bigchaindb.migrations.chain_migration_election import ChainMigrationElection
+from bigchaindb.utils import load_node_key
+from bigchaindb.common.transaction_mode_types import BROADCAST_TX_COMMIT
+from bigchaindb.common.exceptions import (DatabaseDoesNotExist,
+                                          ValidationError)
+from bigchaindb.elections.vote import Vote
 import bigchaindb
-from bigchaindb import backend, processes
+from bigchaindb import (backend, ValidatorElection,
+                        BigchainDB)
 from bigchaindb.backend import schema
-from bigchaindb.backend.admin import (set_replicas, set_shards, add_replicas,
-                                      remove_replicas)
-from bigchaindb.backend.exceptions import OperationError
 from bigchaindb.commands import utils
-from bigchaindb.commands.messages import (
-    CANNOT_START_KEYPAIR_NOT_FOUND,
-    RETHINKDB_STARTUP_ERROR,
-)
-from bigchaindb.commands.utils import (
-    configure_bigchaindb, start_logging_process, input_on_stderr)
-
+from bigchaindb.commands.utils import (configure_bigchaindb,
+                                       input_on_stderr)
+from bigchaindb.log import setup_logging
+from bigchaindb.tendermint_utils import public_key_from_base64
+from bigchaindb.commands.election_types import elections
+from bigchaindb.version import __tm_supported_versions__
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -47,17 +51,12 @@ def run_show_config(args):
     # configure the system.
     config = copy.deepcopy(bigchaindb.config)
     del config['CONFIGURED']
-    private_key = config['keypair']['private']
-    config['keypair']['private'] = 'x' * 45 if private_key else None
     print(json.dumps(config, indent=4, sort_keys=True))
 
 
-def run_configure(args, skip_if_exists=False):
-    """Run a script to configure the current node.
-
-    Args:
-        skip_if_exists (bool): skip the function if a config file already exists
-    """
+@configure_bigchaindb
+def run_configure(args):
+    """Run a script to configure the current node."""
     config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH
 
     config_file_exists = False
@@ -65,9 +64,6 @@ def run_configure(args, skip_if_exists=False):
     if config_path != '-':
         config_file_exists = os.path.exists(config_path)
 
-    if config_file_exists and skip_if_exists:
-        return
-
     if config_file_exists and not args.yes:
         want = input_on_stderr('Config file `{}` exists, do you want to '
                                'override it? (cannot be undone) [y/N]: '.format(config_path))
@@ -76,15 +72,6 @@ def run_configure(args, skip_if_exists=False):
 
     conf = copy.deepcopy(bigchaindb.config)
 
-    # Patch the default configuration with the new values
-    conf = bigchaindb.config_utils.update(
-            conf,
-            bigchaindb.config_utils.env_config(bigchaindb.config))
-
-    print('Generating keypair', file=sys.stderr)
-    conf['keypair']['private'], conf['keypair']['public'] = \
-        crypto.generate_key_pair()
-
     # select the correct config defaults based on the backend
     print('Generating default configuration for backend {}'
           .format(args.backend), file=sys.stderr)
@@ -104,9 +91,9 @@ def run_configure(args, skip_if_exists=False):
             val = conf['database'][key]
             conf['database'][key] = input_on_stderr('Database {}? (default `{}`): '.format(key, val), val)
 
-        val = conf['backlog_reassign_delay']
-        conf['backlog_reassign_delay'] = input_on_stderr(
-            'Stale transaction reassignment delay (in seconds)? (default `{}`): '.format(val), val)
+        for key in ('host', 'port'):
+            val = conf['tendermint'][key]
+            conf['tendermint'][key] = input_on_stderr('Tendermint {}? (default `{}`)'.format(key, val), val)
 
     if config_path != '-':
         bigchaindb.config_utils.write_config(conf, config_path)
@@ -117,41 +104,152 @@ def run_configure(args, skip_if_exists=False):
 
 
 @configure_bigchaindb
-def run_export_my_pubkey(args):
-    """Export this node's public key to standard output
-    """
-    pubkey = bigchaindb.config['keypair']['public']
-    if pubkey is not None:
-        print(pubkey)
+def run_election(args):
+    """Initiate and manage elections"""
+
+    b = BigchainDB()
+
+    # Call the function specified by args.action, as defined above
+    globals()[f'run_election_{args.action}'](args, b)
+
+
+def run_election_new(args, bigchain):
+    election_type = args.election_type.replace('-', '_')
+    globals()[f'run_election_new_{election_type}'](args, bigchain)
+
+
+def create_new_election(sk, bigchain, election_class, data):
+    try:
+        key = load_node_key(sk)
+        voters = election_class.recipients(bigchain)
+        election = election_class.generate([key.public_key],
+                                           voters,
+                                           data, None).sign([key.private_key])
+        election.validate(bigchain)
+    except ValidationError as e:
+        logger.error(e)
+        return False
+    except FileNotFoundError as fd_404:
+        logger.error(fd_404)
+        return False
+
+    resp = bigchain.write_transaction(election, BROADCAST_TX_COMMIT)
+    if resp == (202, ''):
+        logger.info('[SUCCESS] Submitted proposal with id: {}'.format(election.id))
+        return election.id
     else:
-        sys.exit("This node's public key wasn't set anywhere "
-                 "so it can't be exported")
-        # raises SystemExit exception
-        # message is sent to stderr
-        # exits with exit code 1 (signals tha an error happened)
+        logger.error('Failed to commit election proposal')
+        return False
+
+
+def run_election_new_upsert_validator(args, bigchain):
+    """Initiates an election to add/update/remove a validator to an existing BigchainDB network
+
+    :param args: dict
+        args = {
+        'public_key': the public key of the proposed peer, (str)
+        'power': the proposed validator power for the new peer, (str)
+        'node_id': the node_id of the new peer (str)
+        'sk': the path to the private key of the node calling the election (str)
+        }
+    :param bigchain: an instance of BigchainDB
+    :return: election_id or `False` in case of failure
+    """
+
+    new_validator = {
+        'public_key': {'value': public_key_from_base64(args.public_key),
+                       'type': 'ed25519-base16'},
+        'power': args.power,
+        'node_id': args.node_id
+    }
+
+    return create_new_election(args.sk, bigchain, ValidatorElection, new_validator)
+
+
+def run_election_new_chain_migration(args, bigchain):
+    """Initiates an election to halt block production
+
+    :param args: dict
+        args = {
+        'sk': the path to the private key of the node calling the election (str)
+        }
+    :param bigchain: an instance of BigchainDB
+    :return: election_id or `False` in case of failure
+    """
+
+    return create_new_election(args.sk, bigchain, ChainMigrationElection, {})
+
+
+def run_election_approve(args, bigchain):
+    """Approve an election
+
+    :param args: dict
+        args = {
+        'election_id': the election_id of the election (str)
+        'sk': the path to the private key of the signer (str)
+        }
+    :param bigchain: an instance of BigchainDB
+    :return: success log message or `False` in case of error
+    """
+
+    key = load_node_key(args.sk)
+    tx = bigchain.get_transaction(args.election_id)
+    voting_powers = [v.amount for v in tx.outputs if key.public_key in v.public_keys]
+    if len(voting_powers) > 0:
+        voting_power = voting_powers[0]
+    else:
+        logger.error('The key you provided does not match any of the eligible voters in this election.')
+        return False
+
+    inputs = [i for i in tx.to_inputs() if key.public_key in i.owners_before]
+    election_pub_key = ValidatorElection.to_public_key(tx.id)
+    approval = Vote.generate(inputs,
+                             [([election_pub_key], voting_power)],
+                             tx.id).sign([key.private_key])
+    approval.validate(bigchain)
+
+    resp = bigchain.write_transaction(approval, BROADCAST_TX_COMMIT)
+
+    if resp == (202, ''):
+        logger.info('[SUCCESS] Your vote has been submitted')
+        return approval.id
+    else:
+        logger.error('Failed to commit vote')
+        return False
+
+
+def run_election_show(args, bigchain):
+    """Retrieves information about an election
+
+    :param args: dict
+        args = {
+        'election_id': the transaction_id for an election (str)
+        }
+    :param bigchain: an instance of BigchainDB
+    """
+
+    election = bigchain.get_transaction(args.election_id)
+    if not election:
+        logger.error(f'No election found with election_id {args.election_id}')
+        return
+
+    response = election.show_election(bigchain)
+
+    logger.info(response)
+
+    return response
 
 
 def _run_init():
-    # Try to access the keypair, throws an exception if it does not exist
-    b = bigchaindb.Bigchain()
+    bdb = bigchaindb.BigchainDB()
 
-    schema.init_database(connection=b.connection)
-
-    b.create_genesis_block()
-    logger.info('Genesis block created.')
+    schema.init_database(connection=bdb.connection)
 
 
 @configure_bigchaindb
 def run_init(args):
     """Initialize the database"""
-    # TODO Provide mechanism to:
-    # 1. prompt the user to inquire whether they wish to drop the db
-    # 2. force the init, (e.g., via -f flag)
-    try:
-        _run_init()
-    except DatabaseAlreadyExists:
-        print('The database already exists.', file=sys.stderr)
-        print('If you wish to re-initialize it, first drop it.', file=sys.stderr)
+    _run_init()
 
 
 @configure_bigchaindb
@@ -165,90 +263,42 @@ def run_drop(args):
             return
 
     conn = backend.connect()
-    dbname = bigchaindb.config['database']['name']
     try:
         schema.drop_database(conn, dbname)
     except DatabaseDoesNotExist:
         print("Cannot drop '{name}'. The database does not exist.".format(name=dbname), file=sys.stderr)
 
 
+def run_recover(b):
+    rollback(b)
+
+
 @configure_bigchaindb
-@start_logging_process
 def run_start(args):
     """Start the processes to run the node"""
+
+    # Configure Logging
+    setup_logging()
+
     logger.info('BigchainDB Version %s', bigchaindb.__version__)
+    run_recover(bigchaindb.lib.BigchainDB())
 
-    if args.allow_temp_keypair:
-        if not (bigchaindb.config['keypair']['private'] or
-                bigchaindb.config['keypair']['public']):
-
-            private_key, public_key = crypto.generate_key_pair()
-            bigchaindb.config['keypair']['private'] = private_key
-            bigchaindb.config['keypair']['public'] = public_key
-        else:
-            logger.warning('Keypair found, no need to create one on the fly.')
-
-    if args.start_rethinkdb:
-        try:
-            proc = utils.start_rethinkdb()
-        except StartupError as e:
-            sys.exit(RETHINKDB_STARTUP_ERROR.format(e))
-        logger.info('RethinkDB started with PID %s' % proc.pid)
-
-    try:
+    if not args.skip_initialize_database:
+        logger.info('Initializing database')
         _run_init()
-    except DatabaseAlreadyExists:
-        pass
-    except KeypairNotFoundException:
-        sys.exit(CANNOT_START_KEYPAIR_NOT_FOUND)
 
-    logger.info('Starting BigchainDB main process with public key %s',
-                bigchaindb.config['keypair']['public'])
-    processes.start()
+    logger.info('Starting BigchainDB main process.')
+    from bigchaindb.start import start
+    start(args)
 
 
-@configure_bigchaindb
-def run_set_shards(args):
-    conn = backend.connect()
-    try:
-        set_shards(conn, shards=args.num_shards)
-    except OperationError as e:
-        sys.exit(str(e))
-
-
-@configure_bigchaindb
-def run_set_replicas(args):
-    conn = backend.connect()
-    try:
-        set_replicas(conn, replicas=args.num_replicas)
-    except OperationError as e:
-        sys.exit(str(e))
-
-
-@configure_bigchaindb
-def run_add_replicas(args):
-    # Note: This command is specific to MongoDB
-    conn = backend.connect()
-
-    try:
-        add_replicas(conn, args.replicas)
-    except (OperationError, NotImplementedError) as e:
-        sys.exit(str(e))
-    else:
-        print('Added {} to the replicaset.'.format(args.replicas))
-
-
-@configure_bigchaindb
-def run_remove_replicas(args):
-    # Note: This command is specific to MongoDB
-    conn = backend.connect()
-
-    try:
-        remove_replicas(conn, args.replicas)
-    except (OperationError, NotImplementedError) as e:
-        sys.exit(str(e))
-    else:
-        print('Removed {} from the replicaset.'.format(args.replicas))
+def run_tendermint_version(args):
+    """Show the supported Tendermint version(s)"""
+    supported_tm_ver = {
+        'description': 'BigchainDB supports the following Tendermint version(s)',
+        'tendermint': __tm_supported_versions__,
+    }
+    print(json.dumps(supported_tm_ver, indent=4, sort_keys=True))
 
 
 def create_parser():
@@ -265,20 +315,56 @@ def create_parser():
 
     # parser for writing a config file
     config_parser = subparsers.add_parser('configure',
-                                          help='Prepare the config file '
-                                               'and create the node keypair')
+                                          help='Prepare the config file.')
+
     config_parser.add_argument('backend',
-                               choices=['rethinkdb', 'mongodb'],
-                               help='The backend to use. It can be either '
-                                    'rethinkdb or mongodb.')
+                               choices=['localmongodb'],
+                               default='localmongodb',
+                               const='localmongodb',
+                               nargs='?',
+                               help='The backend to use. It can only be '
+                               '"localmongodb", currently.')
+
+    # parser for managing elections
+    election_parser = subparsers.add_parser('election',
+                                            help='Manage elections.')
+
+    election_subparser = election_parser.add_subparsers(title='Action',
+                                                        dest='action')
+
+    new_election_parser = election_subparser.add_parser('new',
+                                                        help='Calls a new election.')
+
+    new_election_subparser = new_election_parser.add_subparsers(title='Election_Type',
+                                                                dest='election_type')
+
+    # Parser factory for each type of new election, so we get a bunch of commands that look like this:
+    # election new <some_election_type> <args>...
+    for name, data in elections.items():
+        args = data['args']
+        generic_parser = new_election_subparser.add_parser(name, help=data['help'])
+        for arg, kwargs in args.items():
+            generic_parser.add_argument(arg, **kwargs)
+
+    approve_election_parser = election_subparser.add_parser('approve',
+                                                            help='Approve the election.')
+    approve_election_parser.add_argument('election_id',
+                                         help='The election_id of the election.')
+    approve_election_parser.add_argument('--private-key',
+                                         dest='sk',
+                                         required=True,
+                                         help='Path to the private key of the election initiator.')
+
+    show_election_parser = election_subparser.add_parser('show',
+                                                         help='Provides information about an election.')
+
+    show_election_parser.add_argument('election_id',
+                                      help='The transaction id of the election you wish to query.')
 
     # parsers for showing/exporting config values
     subparsers.add_parser('show-config',
                           help='Show the current configuration')
 
-    subparsers.add_parser('export-my-pubkey',
-                          help="Export this node's public key")
-
     # parser for database-level commands
     subparsers.add_parser('init',
                           help='Init the database')
@@ -290,57 +376,21 @@ def create_parser():
     start_parser = subparsers.add_parser('start',
                                          help='Start BigchainDB')
 
-    start_parser.add_argument('--dev-allow-temp-keypair',
-                              dest='allow_temp_keypair',
+    start_parser.add_argument('--no-init',
+                              dest='skip_initialize_database',
+                              default=False,
                               action='store_true',
-                              help='Generate a random keypair on start')
+                              help='Skip database initialization')
 
-    start_parser.add_argument('--dev-start-rethinkdb',
-                              dest='start_rethinkdb',
+    subparsers.add_parser('tendermint-version',
+                          help='Show the Tendermint supported versions')
+
+    start_parser.add_argument('--experimental-parallel-validation',
+                              dest='experimental_parallel_validation',
+                              default=False,
                               action='store_true',
-                              help='Run RethinkDB on start')
+                              help='💀 EXPERIMENTAL: parallelize validation for better throughput 💀')
 
-    # parser for configuring the number of shards
-    sharding_parser = subparsers.add_parser('set-shards',
-                                            help='Configure number of shards')
-
-    sharding_parser.add_argument('num_shards', metavar='num_shards',
-                                 type=int, default=1,
-                                 help='Number of shards')
-
-    # parser for configuring the number of replicas
-    replicas_parser = subparsers.add_parser('set-replicas',
-                                            help='Configure number of replicas')
-
-    replicas_parser.add_argument('num_replicas', metavar='num_replicas',
-                                 type=int, default=1,
-                                 help='Number of replicas (i.e. the replication factor)')
-
-    # parser for adding nodes to the replica set
-    add_replicas_parser = subparsers.add_parser('add-replicas',
-                                                help='Add a set of nodes to the '
-                                                     'replica set. This command '
-                                                     'is specific to the MongoDB'
-                                                     ' backend.')
-
-    add_replicas_parser.add_argument('replicas', nargs='+',
-                                     type=utils.mongodb_host,
-                                     help='A list of space separated hosts to '
-                                          'add to the replicaset. Each host '
-                                          'should be in the form `host:port`.')
-
-    # parser for removing nodes from the replica set
-    rm_replicas_parser = subparsers.add_parser('remove-replicas',
-                                               help='Remove a set of nodes from the '
-                                                    'replica set. This command '
-                                                    'is specific to the MongoDB'
-                                                    ' backend.')
-
-    rm_replicas_parser.add_argument('replicas', nargs='+',
-                                    type=utils.mongodb_host,
-                                    help='A list of space separated hosts to '
-                                         'remove from the replicaset. Each host '
-                                         'should be in the form `host:port`.')
     return parser
 
 

bigchaindb/commands/election_types.py

@@ -0,0 +1,31 @@
+elections = {
+    'upsert-validator': {
+        'help': 'Propose a change to the validator set',
+        'args': {
+            'public_key': {
+                'help': 'Public key of the validator to be added/updated/removed.'
+            },
+            'power': {
+                'type': int,
+                'help': 'The proposed power for the validator. Setting to 0 will remove the validator.'},
+            'node_id': {
+                'help': 'The node_id of the validator.'
+            },
+            '--private-key': {
+                'dest': 'sk',
+                'required': True,
+                'help': 'Path to the private key of the election initiator.'
+            }
+        }
+    },
+    'chain-migration': {
+        'help': 'Call for a halt to block production to allow for a version change across breaking changes.',
+        'args': {
+            '--private-key': {
+                'dest': 'sk',
+                'required': True,
+                'help': 'Path to the private key of the election initiator.'
+            }
+        }
+    }
+}

bigchaindb/commands/messages.py

@@ -1,10 +0,0 @@
-"""Module to store messages used in commands, such as error messages,
-warnings, prompts, etc.
-
-"""
-CANNOT_START_KEYPAIR_NOT_FOUND = (
-    "Can't start BigchainDB, no keypair found. "
-    'Did you run `bigchaindb configure`?'
-)
-
-RETHINKDB_STARTUP_ERROR = 'Error starting RethinkDB, reason is: {}'

bigchaindb/commands/utils.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Utility functions and basic common arguments
 for ``argparse.ArgumentParser``.
 """
@@ -6,17 +11,10 @@ import argparse
 import builtins
 import functools
 import multiprocessing as mp
-import subprocess
 import sys
 
-import rethinkdb as r
-from pymongo import uri_parser
-
 import bigchaindb
 import bigchaindb.config_utils
-from bigchaindb import backend
-from bigchaindb.common.exceptions import StartupError
-from bigchaindb.log.setup import setup_logging
 from bigchaindb.version import __version__
 
 
@@ -34,16 +32,18 @@ def configure_bigchaindb(command):
     """
     @functools.wraps(command)
     def configure(args):
+        config_from_cmdline = None
         try:
-            config_from_cmdline = {
-                'log': {
-                    'level_console': args.log_level,
-                    'level_logfile': args.log_level,
-                },
-                'server': {'loglevel': args.log_level},
-            }
+            if args.log_level is not None:
+                config_from_cmdline = {
+                    'log': {
+                        'level_console': args.log_level,
+                        'level_logfile': args.log_level,
+                    },
+                    'server': {'loglevel': args.log_level},
+                }
         except AttributeError:
-            config_from_cmdline = None
+            pass
         bigchaindb.config_utils.autoconfigure(
             filename=args.config, config=config_from_cmdline, force=True)
         command(args)
@@ -51,32 +51,6 @@ def configure_bigchaindb(command):
     return configure
 
 
-def start_logging_process(command):
-    """Decorator to start the logging subscriber process.
-
-    Args:
-        command: The command to decorate.
-
-    Returns:
-        The command wrapper function.
-
-    .. important::
-
-        Configuration, if needed, should be applied before invoking this
-        decorator, as starting the subscriber process for logging will
-        configure the root logger for the child process based on the
-        state of :obj:`bigchaindb.config` at the moment this decorator
-        is invoked.
-
-    """
-    @functools.wraps(command)
-    def start_logging(args):
-        from bigchaindb import config
-        setup_logging(user_log_config=config.get('log'))
-        command(args)
-    return start_logging
-
-
 def _convert(value, default=None, convert=None):
     def convert_bool(value):
         if value.lower() in ('true', 't', 'yes', 'y'):
@@ -123,46 +97,6 @@ def input_on_stderr(prompt='', default=None, convert=None):
     return _convert(value, default, convert)
 
 
-def start_rethinkdb():
-    """Start RethinkDB as a child process and wait for it to be
-    available.
-
-    Raises:
-        :class:`~bigchaindb.common.exceptions.StartupError` if
-            RethinkDB cannot be started.
-    """
-
-    proc = subprocess.Popen(['rethinkdb', '--bind', 'all'],
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.STDOUT,
-                            universal_newlines=True)
-
-    dbname = bigchaindb.config['database']['name']
-    line = ''
-
-    for line in proc.stdout:
-        if line.startswith('Server ready'):
-
-            # FIXME: seems like tables are not ready when the server is ready,
-            #        that's why we need to query RethinkDB to know the state
-            #        of the database. This code assumes the tables are ready
-            #        when the database is ready. This seems a valid assumption.
-            try:
-                conn = backend.connect()
-                # Before checking if the db is ready, we need to query
-                # the server to check if it contains that db
-                if conn.run(r.db_list().contains(dbname)):
-                    conn.run(r.db(dbname).wait())
-            except (r.ReqlOpFailedError, r.ReqlDriverError) as exc:
-                raise StartupError('Error waiting for the database `{}` '
-                                   'to be ready'.format(dbname)) from exc
-            return proc
-
-    # We are here when we exhaust the stdout of the process.
-    # The last `line` contains info about the error.
-    raise StartupError(line)
-
-
 def start(parser, argv, scope):
     """Utility function to execute a subcommand.
 
@@ -204,44 +138,17 @@ def start(parser, argv, scope):
     return func(args)
 
 
-def mongodb_host(host):
-    """Utility function that works as a type for mongodb ``host`` args.
-
-    This function validates the ``host`` args provided by to the
-    ``add-replicas`` and ``remove-replicas`` commands and checks if each arg
-    is in the form "host:port"
-
-    Args:
-        host (str): A string containing hostname and port (e.g. "host:port")
-
-    Raises:
-        ArgumentTypeError: if it fails to parse the argument
-    """
-    # check if mongodb can parse the host
-    try:
-        hostname, port = uri_parser.parse_host(host, default_port=None)
-    except ValueError as exc:
-        raise argparse.ArgumentTypeError(exc.args[0])
-
-    # we do require the port to be provided.
-    if port is None or hostname == '':
-        raise argparse.ArgumentTypeError('expected host in the form '
-                                         '`host:port`. Got `{}` instead.'
-                                         .format(host))
-
-    return host
-
-
 base_parser = argparse.ArgumentParser(add_help=False, prog='bigchaindb')
 
 base_parser.add_argument('-c', '--config',
                          help='Specify the location of the configuration file '
                               '(use "-" for stdout)')
 
+# NOTE: this flag should not have any default value because that will override
+# the environment variables provided to configure the logger.
 base_parser.add_argument('-l', '--log-level',
                          type=str.upper,  # convert to uppercase for comparison to choices
-                         choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
-                         default='INFO',
+                         choices=['DEBUG', 'BENCHMARK', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
                          help='Log level')
 
 base_parser.add_argument('-y', '--yes', '--yes-please',

bigchaindb/common/crypto.py

@@ -1,7 +1,16 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 # Separate all crypto code so that we can easily test several implementations
 from collections import namedtuple
 
-import sha3
+try:
+    from hashlib import sha3_256
+except ImportError:
+    from sha3 import sha3_256
+
 from cryptoconditions import crypto
 
 
@@ -10,7 +19,7 @@ CryptoKeypair = namedtuple('CryptoKeypair', ('private_key', 'public_key'))
 
 def hash_data(data):
     """Hash the provided data using SHA3-256"""
-    return sha3.sha3_256(data.encode()).hexdigest()
+    return sha3_256(data.encode()).hexdigest()
 
 
 def generate_key_pair():
@@ -30,3 +39,17 @@ def generate_key_pair():
 
 PrivateKey = crypto.Ed25519SigningKey
 PublicKey = crypto.Ed25519VerifyingKey
+
+
+def key_pair_from_ed25519_key(hex_private_key):
+    """Generate base58 encode public-private key pair from a hex encoded private key"""
+    priv_key = crypto.Ed25519SigningKey(bytes.fromhex(hex_private_key)[:32], encoding='bytes')
+    public_key = priv_key.get_verifying_key()
+    return CryptoKeypair(private_key=priv_key.encode(encoding='base58').decode('utf-8'),
+                         public_key=public_key.encode(encoding='base58').decode('utf-8'))
+
+
+def public_key_from_ed25519_key(hex_public_key):
+    """Generate base58 public key from hex encoded public key"""
+    public_key = crypto.Ed25519VerifyingKey(bytes.fromhex(hex_public_key), encoding='bytes')
+    return public_key.encode(encoding='base58').decode('utf-8')

bigchaindb/common/exceptions.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Custom exceptions used in the `bigchaindb` package.
 """
 from bigchaindb.exceptions import BigchainDBError
@@ -7,10 +12,6 @@ class ConfigurationError(BigchainDBError):
     """Raised when there is a problem with server configuration"""
 
 
-class DatabaseAlreadyExists(BigchainDBError):
-    """Raised when trying to create the database but the db is already there"""
-
-
 class DatabaseDoesNotExist(BigchainDBError):
     """Raised when trying to delete the database but the db is not there"""
 
@@ -23,13 +24,10 @@ class CyclicBlockchainError(BigchainDBError):
     """Raised when there is a cycle in the blockchain"""
 
 
-class KeypairNotFoundException(BigchainDBError):
-    """Raised if operation cannot proceed because the keypair was not given"""
-
-
 class KeypairMismatchException(BigchainDBError):
     """Raised if the private key(s) provided for signing don't match any of the
-    current owner(s)"""
+    current owner(s)
+    """
 
 
 class OperationError(BigchainDBError):
@@ -55,7 +53,8 @@ class DoubleSpend(ValidationError):
 
 class InvalidHash(ValidationError):
     """Raised if there was an error checking the hash for a particular
-    operation"""
+    operation
+    """
 
 
 class SchemaValidationError(ValidationError):
@@ -64,20 +63,8 @@ class SchemaValidationError(ValidationError):
 
 class InvalidSignature(ValidationError):
     """Raised if there was an error checking the signature for a particular
-    operation"""
-
-
-class ImproperVoteError(ValidationError):
-    """Raised if a vote is not constructed correctly, or signed incorrectly"""
-
-
-class MultipleVotesError(ValidationError):
-    """Raised if a voter has voted more than once"""
-
-
-class TransactionNotInValidBlock(ValidationError):
-    """Raised when a transfer transaction is attempting to fulfill the
-    outputs of a transaction that is in an invalid or undecided block"""
+    operation
+    """
 
 
 class AssetIdMismatch(ValidationError):
@@ -96,10 +83,6 @@ class TransactionOwnerError(ValidationError):
     """Raised if a user tries to transfer a transaction they don't own"""
 
 
-class SybilError(ValidationError):
-    """If a block or vote comes from an unidentifiable node"""
-
-
 class DuplicateTransaction(ValidationError):
     """Raised if a duplicated transaction is found"""
 
@@ -108,5 +91,25 @@ class ThresholdTooDeep(ValidationError):
     """Raised if threshold condition is too deep"""
 
 
-class GenesisBlockAlreadyExistsError(ValidationError):
-    """Raised when trying to create the already existing genesis block"""
+class MultipleValidatorOperationError(ValidationError):
+    """Raised when a validator update pending but new request is submited"""
+
+
+class MultipleInputsError(ValidationError):
+    """Raised if there were multiple inputs when only one was expected"""
+
+
+class InvalidProposer(ValidationError):
+    """Raised if the public key is not a part of the validator set"""
+
+
+class UnequalValidatorSet(ValidationError):
+    """Raised if the validator sets differ"""
+
+
+class InvalidPowerChange(ValidationError):
+    """Raised if proposed power change in validator set is >=1/3 total power"""
+
+
+class InvalidPublicKey(ValidationError):
+    """Raised if public key doesn't match the encoding type"""

bigchaindb/common/memoize.py

@@ -0,0 +1,58 @@
+import functools
+import codecs
+from functools import lru_cache
+
+
+class HDict(dict):
+    def __hash__(self):
+        return hash(codecs.decode(self['id'], 'hex'))
+
+
+@lru_cache(maxsize=16384)
+def from_dict(func, *args, **kwargs):
+    return func(*args, **kwargs)
+
+
+def memoize_from_dict(func):
+
+    @functools.wraps(func)
+    def memoized_func(*args, **kwargs):
+
+        if args[1].get('id', None):
+            args = list(args)
+            args[1] = HDict(args[1])
+            new_args = tuple(args)
+            return from_dict(func, *new_args, **kwargs)
+        else:
+            return func(*args, **kwargs)
+
+    return memoized_func
+
+
+class ToDictWrapper():
+    def __init__(self, tx):
+        self.tx = tx
+
+    def __eq__(self, other):
+        return self.tx.id == other.tx.id
+
+    def __hash__(self):
+        return hash(self.tx.id)
+
+
+@lru_cache(maxsize=16384)
+def to_dict(func, tx_wrapped):
+    return func(tx_wrapped.tx)
+
+
+def memoize_to_dict(func):
+
+    @functools.wraps(func)
+    def memoized_func(*args, **kwargs):
+
+        if args[0].id:
+            return to_dict(func, ToDictWrapper(args[0]))
+        else:
+            return func(*args, **kwargs)
+
+    return memoized_func

bigchaindb/common/schema/README.md

@@ -1,13 +1,37 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+BigchainDB and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Introduction
 
 This directory contains the schemas for the different JSON documents BigchainDB uses.
 
 The aim is to provide:
- - a strict definition/documentation of the data structures used in BigchainDB
- - a language independent tool to validate the structure of incoming/outcoming
-   data (there are several ready to use
-   [implementations](http://json-schema.org/implementations.html) written in
-   different languages)
+
+- a strict definition of the data structures used in BigchainDB,
+- a language-independent tool to validate the structure of incoming/outcoming
+  data. (There are several ready to use
+  [implementations](http://json-schema.org/implementations.html) written in
+  different languages.)
+
+## Sources
+
+The files defining the JSON Schema for transactions (`transaction_*.yaml`)
+are based on the [BigchainDB Transactions Specs](https://github.com/bigchaindb/BEPs/tree/master/tx-specs).
+If you want to add a new transaction version,
+you must write a spec for it first.
+(You can't change the JSON Schema files for old versions.
+Those were used to validate old transactions
+and are needed to re-check those transactions.)
+
+There used to be a file defining the JSON Schema for votes, named `vote.yaml`.
+It was used by BigchainDB version 1.3.0 and earlier.
+If you want a copy of the latest `vote.yaml` file,
+then you can get it from the version 1.3.0 release on GitHub, at
+[https://github.com/bigchaindb/bigchaindb/blob/v1.3.0/bigchaindb/common/schema/vote.yaml](https://github.com/bigchaindb/bigchaindb/blob/v1.3.0/bigchaindb/common/schema/vote.yaml).
 
 ## Learn about JSON Schema
 

bigchaindb/common/schema/__init__.py

@@ -1,11 +1,15 @@
-""" Schema validation related functions and data """
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+"""Schema validation related functions and data"""
 import os.path
 import logging
 
 import jsonschema
 import yaml
 import rapidjson
-import rapidjson_schema
 
 from bigchaindb.common.exceptions import SchemaValidationError
 
@@ -13,36 +17,35 @@ from bigchaindb.common.exceptions import SchemaValidationError
 logger = logging.getLogger(__name__)
 
 
-def drop_schema_descriptions(node):
-    """ Drop descriptions from schema, since they clutter log output """
-    if 'description' in node:
-        del node['description']
-    for n in node.get('properties', {}).values():
-        drop_schema_descriptions(n)
-    for n in node.get('definitions', {}).values():
-        drop_schema_descriptions(n)
-    for n in node.get('anyOf', []):
-        drop_schema_descriptions(n)
-
-
-def _load_schema(name):
-    """ Load a schema from disk """
-    path = os.path.join(os.path.dirname(__file__), name + '.yaml')
+def _load_schema(name, path=__file__):
+    """Load a schema from disk"""
+    path = os.path.join(os.path.dirname(path), name + '.yaml')
     with open(path) as handle:
         schema = yaml.safe_load(handle)
-    drop_schema_descriptions(schema)
-    fast_schema = rapidjson_schema.loads(rapidjson.dumps(schema))
+    fast_schema = rapidjson.Validator(rapidjson.dumps(schema))
     return path, (schema, fast_schema)
 
 
-TX_SCHEMA_PATH, TX_SCHEMA_COMMON = _load_schema('transaction')
-_, TX_SCHEMA_CREATE = _load_schema('transaction_create')
-_, TX_SCHEMA_TRANSFER = _load_schema('transaction_transfer')
-VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote')
+TX_SCHEMA_VERSION = 'v2.0'
+
+TX_SCHEMA_PATH, TX_SCHEMA_COMMON = _load_schema('transaction_' +
+                                                TX_SCHEMA_VERSION)
+_, TX_SCHEMA_CREATE = _load_schema('transaction_create_' +
+                                   TX_SCHEMA_VERSION)
+_, TX_SCHEMA_TRANSFER = _load_schema('transaction_transfer_' +
+                                     TX_SCHEMA_VERSION)
+
+_, TX_SCHEMA_VALIDATOR_ELECTION = _load_schema('transaction_validator_election_' +
+                                               TX_SCHEMA_VERSION)
+
+_, TX_SCHEMA_CHAIN_MIGRATION_ELECTION = _load_schema('transaction_chain_migration_election_' +
+                                                     TX_SCHEMA_VERSION)
+
+_, TX_SCHEMA_VOTE = _load_schema('transaction_vote_' + TX_SCHEMA_VERSION)
 
 
 def _validate_schema(schema, body):
-    """ Validate data against a schema """
+    """Validate data against a schema"""
 
     # Note
     #
@@ -56,7 +59,7 @@ def _validate_schema(schema, body):
     # a helpful error message.
 
     try:
-        schema[1].validate(rapidjson.dumps(body))
+        schema[1](rapidjson.dumps(body))
     except ValueError as exc:
         try:
             jsonschema.validate(body, schema[0])
@@ -67,8 +70,7 @@ def _validate_schema(schema, body):
 
 
 def validate_transaction_schema(tx):
-    """
-    Validate a transaction dict.
+    """Validate a transaction dict.
 
     TX_SCHEMA_COMMON contains properties that are common to all types of
     transaction. TX_SCHEMA_[TRANSFER|CREATE] add additional constraints on top.
@@ -78,8 +80,3 @@ def validate_transaction_schema(tx):
         _validate_schema(TX_SCHEMA_TRANSFER, tx)
     else:
         _validate_schema(TX_SCHEMA_CREATE, tx)
-
-
-def validate_vote_schema(vote):
-    """ Validate a vote dict """
-    _validate_schema(VOTE_SCHEMA, vote)

bigchaindb/common/schema/transaction.yaml

@@ -1,247 +0,0 @@
----
-"$schema": "http://json-schema.org/draft-04/schema#"
-id: "http://www.bigchaindb.com/schema/transaction.json"
-type: object
-additionalProperties: false
-title: Transaction Schema
-description: |
-    A transaction represents the creation or transfer of assets in BigchainDB.
-required:
-- id
-- inputs
-- outputs
-- operation
-- metadata
-- asset
-- version
-properties:
-  id:
-    "$ref": "#/definitions/sha3_hexdigest"
-    description: |
-        A sha3 digest of the transaction. The ID is calculated by removing all
-        derived hashes and signatures from the transaction, serializing it to
-        JSON with keys in sorted order and then hashing the resulting string
-        with sha3.
-  operation:
-    "$ref": "#/definitions/operation"
-  asset:
-    "$ref": "#/definitions/asset"
-    description: |
-      Description of the asset being transacted.
-
-      See: `Asset`_.
-  inputs:
-    type: array
-    title: "Transaction inputs"
-    description: |
-        Array of the inputs of a transaction.
-
-        See: Input_.
-    items:
-      "$ref": "#/definitions/input"
-  outputs:
-    type: array
-    description: |
-        Array of outputs provided by this transaction.
-
-        See: Output_.
-    items:
-      "$ref": "#/definitions/output"
-  metadata:
-    "$ref": "#/definitions/metadata"
-    description: |
-      User provided transaction metadata. This field may be ``null`` or may
-      contain an id and an object with freeform metadata.
-
-      See: `Metadata`_.
-  version:
-    type: string
-    pattern: "^1\\.0$"
-    description: |
-        BigchainDB transaction schema version.
-definitions:
-  offset:
-    type: integer
-    minimum: 0
-  base58:
-    pattern: "[1-9a-zA-Z^OIl]{43,44}"
-    type: string
-  public_keys:
-    anyOf:
-    - type: array
-      items:
-        "$ref": "#/definitions/base58"
-    - type: 'null'
-  sha3_hexdigest:
-    pattern: "[0-9a-f]{64}"
-    type: string
-  uuid4:
-    pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}"
-    type: string
-    description: |
-        A `UUID <https://tools.ietf.org/html/rfc4122.html>`_
-        of type 4 (random).
-  operation:
-    type: string
-    description: |
-        Type of the transaction:
-
-        A ``CREATE`` transaction creates an asset in BigchainDB. This
-        transaction has outputs but no inputs, so a dummy input is created.
-
-        A ``TRANSFER`` transaction transfers ownership of an asset, by providing
-        an input that meets the conditions of an earlier transaction's outputs.
-
-        A ``GENESIS`` transaction is a special case transaction used as the
-        sole member of the first block in a BigchainDB ledger.
-    enum:
-    - CREATE
-    - TRANSFER
-    - GENESIS
-  asset:
-    type: object
-    description: |
-        Description of the asset being transacted. In the case of a ``TRANSFER``
-        transaction, this field contains only the ID of asset. In the case
-        of a ``CREATE`` transaction, this field contains only the user-defined
-        payload.
-    additionalProperties: false
-    properties:
-      id:
-        "$ref": "#/definitions/sha3_hexdigest"
-        description: |
-          ID of the transaction that created the asset.
-      data:
-        description: |
-          User provided metadata associated with the asset. May also be ``null``.
-        anyOf:
-        - type: object
-          additionalProperties: true
-        - type: 'null'
-  output:
-    type: object
-    description: |
-        A transaction output. Describes the quantity of an asset and the
-        requirements that must be met to spend the output.
-
-        See also: Input_.
-    additionalProperties: false
-    required:
-    - amount
-    - condition
-    - public_keys
-    properties:
-      amount:
-        type: string
-        pattern: "^[0-9]{1,20}$"
-        description: |
-            Integral amount of the asset represented by this output.
-            In the case of a non divisible asset, this will always be 1.
-      condition:
-        description: |
-            Describes the condition that needs to be met to spend the output. Has the properties: 
-
-            - **details**: Details of the condition.
-            - **uri**: Condition encoded as an ASCII string.
-        type: object
-        additionalProperties: false
-        required:
-        - details
-        - uri
-        properties:
-          details:
-            "$ref": "#/definitions/condition_details"
-          uri:
-            type: string
-            pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\
-              (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\
-              subtypes=ed25519-sha-256(&)?){2,3}$"
-      public_keys:
-        "$ref": "#/definitions/public_keys"
-        description: |
-            List of public keys associated with the conditions on an output.
-  input:
-    type: "object"
-    description:
-      An input spends a previous output, by providing one or more fulfillments
-      that fulfill the conditions of the previous output.
-    additionalProperties: false
-    required:
-    - owners_before
-    - fulfillment
-    properties:
-      owners_before:
-        "$ref": "#/definitions/public_keys"
-        description: |
-            List of public keys of the previous owners of the asset.
-      fulfillment:
-        description: |
-          Fulfillment of an `Output.condition`_, or, put a different way, a payload
-          that satisfies the condition of a previous output to prove that the
-          creator(s) of this transaction have control over the listed asset.
-        anyOf:
-        - type: string
-          pattern: "^[a-zA-Z0-9_-]*$"
-        - "$ref": "#/definitions/condition_details"
-      fulfills:
-        anyOf:
-        - type: 'object'
-          description: |
-            Reference to the output that is being spent.
-          additionalProperties: false
-          required:
-          - output_index
-          - transaction_id
-          properties:
-            output_index:
-              "$ref": "#/definitions/offset"
-              description: |
-                Index of the output containing the condition being fulfilled
-            transaction_id:
-              "$ref": "#/definitions/sha3_hexdigest"
-              description: |
-                Transaction ID containing the output to spend
-        - type: 'null'
-  metadata:
-    anyOf:
-    - type: object
-      description: |
-        User provided transaction metadata. This field may be ``null`` or may
-        contain an non empty object with freeform metadata.
-      additionalProperties: true
-      minProperties: 1
-    - type: 'null'
-  condition_details:
-    description: |
-      Details needed to reconstruct the condition associated with an output.
-      Currently, BigchainDB only supports ed25519 and threshold condition types.
-    anyOf:
-    - type: object
-      additionalProperties: false
-      required:
-      - type
-      - public_key
-      properties:
-        type:
-          type: string
-          pattern: "^ed25519-sha-256$"
-        public_key:
-          "$ref": "#/definitions/base58"
-    - type: object
-      additionalProperties: false
-      required:
-      - type
-      - threshold
-      - subconditions
-      properties:
-        type:
-          type: "string"
-          pattern: "^threshold-sha-256$"
-        threshold:
-          type: integer
-          minimum: 1
-          maximum: 100
-        subconditions:
-          type: array
-          items:
-            "$ref": "#/definitions/condition_details"

bigchaindb/common/schema/transaction_chain_migration_election_v2.0.yaml

@@ -0,0 +1,45 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+title: Chain Migration Election Schema - Propose a halt in block production to allow for a version change
+required:
+- operation
+- asset
+- outputs
+properties:
+  operation:
+    type: string
+    value: "CHAIN_MIGRATION_ELECTION"
+  asset:
+    additionalProperties: false
+    properties:
+      data:
+        additionalProperties: false
+        properties:
+          seed:
+            type: string
+    required:
+    - data
+  outputs:
+    type: array
+    items:
+      "$ref": "#/definitions/output"
+definitions:
+  output:
+    type: object
+    properties:
+      condition:
+        type: object
+        required:
+          - uri
+        properties:
+          uri:
+            type: string
+            pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\
+              (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\
+              subtypes=ed25519-sha-256(&)?){2,3}$"

bigchaindb/common/schema/transaction_create_v1.0.yaml

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 ---
 "$schema": "http://json-schema.org/draft-04/schema#"
 type: object

bigchaindb/common/schema/transaction_create_v2.0.yaml

@@ -0,0 +1,35 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+title: Transaction Schema - CREATE specific constraints
+required:
+- asset
+- inputs
+properties:
+  asset:
+    additionalProperties: false
+    properties:
+      data:
+        anyOf:
+        - type: object
+          additionalProperties: true
+        - type: 'null'
+    required:
+    - data
+  inputs:
+    type: array
+    title: "Transaction inputs"
+    maxItems: 1
+    minItems: 1
+    items:
+      type: "object"
+      required:
+      - fulfills
+      properties:
+        fulfills:
+            type: "null"

bigchaindb/common/schema/transaction_transfer_v1.0.yaml

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 ---
 "$schema": "http://json-schema.org/draft-04/schema#"
 type: object
@@ -10,8 +15,6 @@ properties:
     properties:
       id:
         "$ref": "#/definitions/sha3_hexdigest"
-        description: |
-          ID of the transaction that created the asset.
     required:
     - id
   inputs:

bigchaindb/common/schema/transaction_transfer_v2.0.yaml

@@ -0,0 +1,34 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+title: Transaction Schema - TRANSFER specific properties
+required:
+- asset
+properties:
+  asset:
+    additionalProperties: false
+    properties:
+      id:
+        "$ref": "#/definitions/sha3_hexdigest"
+    required:
+    - id
+  inputs:
+    type: array
+    title: "Transaction inputs"
+    minItems: 1
+    items:
+      type: "object"
+      required:
+      - fulfills
+      properties:
+        fulfills:
+            type: "object"
+definitions:
+  sha3_hexdigest:
+    pattern: "[0-9a-f]{64}"
+    type: string

bigchaindb/common/schema/transaction_v1.0.yaml

@@ -0,0 +1,168 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+additionalProperties: false
+title: Transaction Schema
+required:
+- id
+- inputs
+- outputs
+- operation
+- metadata
+- asset
+- version
+properties:
+  id:
+    anyOf:
+    - "$ref": "#/definitions/sha3_hexdigest"
+    - type: 'null'
+  operation:
+    "$ref": "#/definitions/operation"
+  asset:
+    "$ref": "#/definitions/asset"
+  inputs:
+    type: array
+    title: "Transaction inputs"
+    items:
+      "$ref": "#/definitions/input"
+  outputs:
+    type: array
+    items:
+      "$ref": "#/definitions/output"
+  metadata:
+    "$ref": "#/definitions/metadata"
+  version:
+    type: string
+    pattern: "^1\\.0$"
+definitions:
+  offset:
+    type: integer
+    minimum: 0
+  base58:
+    pattern: "[1-9a-zA-Z^OIl]{43,44}"
+    type: string
+  public_keys:
+    anyOf:
+    - type: array
+      items:
+        "$ref": "#/definitions/base58"
+    - type: 'null'
+  sha3_hexdigest:
+    pattern: "[0-9a-f]{64}"
+    type: string
+  uuid4:
+    pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}"
+    type: string
+  operation:
+    type: string
+    enum:
+    - CREATE
+    - TRANSFER
+    - GENESIS
+  asset:
+    type: object
+    additionalProperties: false
+    properties:
+      id:
+        "$ref": "#/definitions/sha3_hexdigest"
+      data:
+        anyOf:
+        - type: object
+          additionalProperties: true
+        - type: 'null'
+  output:
+    type: object
+    additionalProperties: false
+    required:
+    - amount
+    - condition
+    - public_keys
+    properties:
+      amount:
+        type: string
+        pattern: "^[0-9]{1,20}$"
+      condition:
+        type: object
+        additionalProperties: false
+        required:
+        - details
+        - uri
+        properties:
+          details:
+            "$ref": "#/definitions/condition_details"
+          uri:
+            type: string
+            pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\
+              (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\
+              subtypes=ed25519-sha-256(&)?){2,3}$"
+      public_keys:
+        "$ref": "#/definitions/public_keys"
+  input:
+    type: "object"
+    additionalProperties: false
+    required:
+    - owners_before
+    - fulfillment
+    properties:
+      owners_before:
+        "$ref": "#/definitions/public_keys"
+      fulfillment:
+        anyOf:
+        - type: string
+          pattern: "^[a-zA-Z0-9_-]*$"
+        - "$ref": "#/definitions/condition_details"
+      fulfills:
+        anyOf:
+        - type: 'object'
+          additionalProperties: false
+          required:
+          - output_index
+          - transaction_id
+          properties:
+            output_index:
+              "$ref": "#/definitions/offset"
+            transaction_id:
+              "$ref": "#/definitions/sha3_hexdigest"
+        - type: 'null'
+  metadata:
+    anyOf:
+    - type: object
+      additionalProperties: true
+      minProperties: 1
+    - type: 'null'
+  condition_details:
+    anyOf:
+    - type: object
+      additionalProperties: false
+      required:
+      - type
+      - public_key
+      properties:
+        type:
+          type: string
+          pattern: "^ed25519-sha-256$"
+        public_key:
+          "$ref": "#/definitions/base58"
+    - type: object
+      additionalProperties: false
+      required:
+      - type
+      - threshold
+      - subconditions
+      properties:
+        type:
+          type: "string"
+          pattern: "^threshold-sha-256$"
+        threshold:
+          type: integer
+          minimum: 1
+          maximum: 100
+        subconditions:
+          type: array
+          items:
+            "$ref": "#/definitions/condition_details"

bigchaindb/common/schema/transaction_v2.0.yaml

@@ -0,0 +1,170 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+additionalProperties: false
+title: Transaction Schema
+required:
+- id
+- inputs
+- outputs
+- operation
+- metadata
+- asset
+- version
+properties:
+  id:
+    anyOf:
+    - "$ref": "#/definitions/sha3_hexdigest"
+    - type: 'null'
+  operation:
+    "$ref": "#/definitions/operation"
+  asset:
+    "$ref": "#/definitions/asset"
+  inputs:
+    type: array
+    title: "Transaction inputs"
+    items:
+      "$ref": "#/definitions/input"
+  outputs:
+    type: array
+    items:
+      "$ref": "#/definitions/output"
+  metadata:
+    "$ref": "#/definitions/metadata"
+  version:
+    type: string
+    pattern: "^2\\.0$"
+definitions:
+  offset:
+    type: integer
+    minimum: 0
+  base58:
+    pattern: "[1-9a-zA-Z^OIl]{43,44}"
+    type: string
+  public_keys:
+    anyOf:
+    - type: array
+      items:
+        "$ref": "#/definitions/base58"
+    - type: 'null'
+  sha3_hexdigest:
+    pattern: "[0-9a-f]{64}"
+    type: string
+  uuid4:
+    pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}"
+    type: string
+  operation:
+    type: string
+    enum:
+    - CREATE
+    - TRANSFER
+    - VALIDATOR_ELECTION
+    - CHAIN_MIGRATION_ELECTION
+    - VOTE
+  asset:
+    type: object
+    additionalProperties: false
+    properties:
+      id:
+        "$ref": "#/definitions/sha3_hexdigest"
+      data:
+        anyOf:
+        - type: object
+          additionalProperties: true
+        - type: 'null'
+  output:
+    type: object
+    additionalProperties: false
+    required:
+    - amount
+    - condition
+    - public_keys
+    properties:
+      amount:
+        type: string
+        pattern: "^[0-9]{1,20}$"
+      condition:
+        type: object
+        additionalProperties: false
+        required:
+        - details
+        - uri
+        properties:
+          details:
+            "$ref": "#/definitions/condition_details"
+          uri:
+            type: string
+            pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\
+              (fpt=(ed25519|threshold)-sha-256(&)?|cost=[0-9]+(&)?|\
+              subtypes=ed25519-sha-256(&)?){2,3}$"
+      public_keys:
+        "$ref": "#/definitions/public_keys"
+  input:
+    type: "object"
+    additionalProperties: false
+    required:
+    - owners_before
+    - fulfillment
+    properties:
+      owners_before:
+        "$ref": "#/definitions/public_keys"
+      fulfillment:
+        anyOf:
+        - type: string
+          pattern: "^[a-zA-Z0-9_-]*$"
+        - "$ref": "#/definitions/condition_details"
+      fulfills:
+        anyOf:
+        - type: 'object'
+          additionalProperties: false
+          required:
+          - output_index
+          - transaction_id
+          properties:
+            output_index:
+              "$ref": "#/definitions/offset"
+            transaction_id:
+              "$ref": "#/definitions/sha3_hexdigest"
+        - type: 'null'
+  metadata:
+    anyOf:
+    - type: object
+      additionalProperties: true
+      minProperties: 1
+    - type: 'null'
+  condition_details:
+    anyOf:
+    - type: object
+      additionalProperties: false
+      required:
+      - type
+      - public_key
+      properties:
+        type:
+          type: string
+          pattern: "^ed25519-sha-256$"
+        public_key:
+          "$ref": "#/definitions/base58"
+    - type: object
+      additionalProperties: false
+      required:
+      - type
+      - threshold
+      - subconditions
+      properties:
+        type:
+          type: "string"
+          pattern: "^threshold-sha-256$"
+        threshold:
+          type: integer
+          minimum: 1
+          maximum: 100
+        subconditions:
+          type: array
+          items:
+            "$ref": "#/definitions/condition_details"

bigchaindb/common/schema/transaction_validator_election_v2.0.yaml

@@ -0,0 +1,68 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+title: Validator Election Schema - Propose a change to validator set
+required:
+- operation
+- asset
+- outputs
+properties:
+  operation:
+    type: string
+    value: "VALIDATOR_ELECTION"
+  asset:
+    additionalProperties: false
+    properties:
+      data:
+        additionalProperties: false
+        properties:
+          node_id:
+            type: string
+          seed:
+            type: string
+          public_key:
+            type: object
+            additionalProperties: false
+            required:
+            - value
+            - type
+            properties:
+              value:
+                type: string
+              type:
+                type: string
+                enum:
+                  - ed25519-base16
+                  - ed25519-base32
+                  - ed25519-base64
+          power:
+            "$ref": "#/definitions/positiveInteger"
+        required:
+          - node_id
+          - public_key
+          - power
+    required:
+    - data
+  outputs:
+    type: array
+    items:
+      "$ref": "#/definitions/output"
+definitions:
+  output:
+    type: object
+    properties:
+      condition:
+        type: object
+        required:
+          - uri
+        properties:
+          uri:
+            type: string
+            pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\
+              (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\
+              subtypes=ed25519-sha-256(&)?){2,3}$"

bigchaindb/common/schema/transaction_vote_v2.0.yaml

@@ -0,0 +1,34 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+---
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: object
+title: Vote Schema - Vote on an election
+required:
+- operation
+- outputs
+properties:
+  operation:
+    type: string
+    value: "VOTE"
+  outputs:
+    type: array
+    items:
+      "$ref": "#/definitions/output"
+definitions:
+  output:
+    type: object
+    properties:
+      condition:
+        type: object
+        required:
+          - uri
+        properties:
+          uri:
+            type: string
+            pattern: "^ni:///sha-256;([a-zA-Z0-9_-]{0,86})[?]\
+              (fpt=ed25519-sha-256(&)?|cost=[0-9]+(&)?|\
+              subtypes=ed25519-sha-256(&)?){2,3}$"

bigchaindb/common/schema/vote.yaml

@@ -1,73 +0,0 @@
----
-"$schema": "http://json-schema.org/draft-04/schema#"
-id: "http://www.bigchaindb.com/schema/vote.json"
-type: object
-additionalProperties: false
-title: Vote Schema
-description: |
-    A Vote is an endorsement of a Block (identified by a hash) by
-    a node (identified by a public key).
-
-    The outer Vote object contains the details of the vote being made
-    as well as the signature and identifying information of the node
-    passing the vote.
-required:
-- node_pubkey
-- signature
-- vote
-properties:
-  node_pubkey:
-    type: "string"
-    pattern: "[1-9a-zA-Z^OIl]{43,44}"
-    description: |
-        Ed25519 public key identifying the voting node.
-  signature:
-    type: "string"
-    pattern: "[1-9a-zA-Z^OIl]{86,88}"
-    description:
-        Ed25519 signature of the `Vote Details`_ object.
-  vote:
-    type: "object"
-    additionalProperties: false
-    description: |
-        `Vote Details`_ to be signed.
-    required:
-    - invalid_reason
-    - is_block_valid
-    - previous_block
-    - voting_for_block
-    - timestamp
-    properties:
-      previous_block:
-        "$ref": "#/definitions/sha3_hexdigest"
-        description: |
-          ID (SHA3 hash) of the block that precedes the block being voted on.
-          The notion of a "previous" block is subject to vote.
-      voting_for_block:
-        "$ref": "#/definitions/sha3_hexdigest"
-        description: |
-          ID (SHA3 hash) of the block being voted on.
-      is_block_valid:
-        type: "boolean"
-        description: |
-          This field is ``true`` if the block was deemed valid by the node.
-      invalid_reason:
-        anyOf:
-        - type: "string"
-          description: |
-            Reason the block is voted invalid, or ``null``.
-
-            .. container:: notice
-
-              **Note**: The invalid_reason was not being used and may be dropped in a future version of BigchainDB. See Issue `#217 <https://github.com/bigchaindb/bigchaindb/issues/217>`_ on GitHub.
-        - type: "null"
-      timestamp:
-        type: "string"
-        pattern: "[0-9]{10}"
-        description: |
-          Unix timestamp that the vote was created by the node, according
-          to the system time of the node.
-definitions:
-  sha3_hexdigest:
-    pattern: "[0-9a-f]{64}"
-    type: string

bigchaindb/common/transaction.py

@@ -1,21 +1,51 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Transaction related models to parse and construct transaction
 payloads.
 
+Attributes:
+    UnspentOutput (namedtuple): Object holding the information
+        representing an unspent output.
+
 """
+from collections import namedtuple
 from copy import deepcopy
-from functools import reduce
+from functools import reduce, lru_cache
+import rapidjson
 
 import base58
 from cryptoconditions import Fulfillment, ThresholdSha256, Ed25519Sha256
 from cryptoconditions.exceptions import (
     ParsingError, ASN1DecodeError, ASN1EncodeError, UnsupportedTypeError)
+try:
+    from hashlib import sha3_256
+except ImportError:
+    from sha3 import sha3_256
 
 from bigchaindb.common.crypto import PrivateKey, hash_data
 from bigchaindb.common.exceptions import (KeypairMismatchException,
+                                          InputDoesNotExist, DoubleSpend,
                                           InvalidHash, InvalidSignature,
                                           AmountError, AssetIdMismatch,
                                           ThresholdTooDeep)
 from bigchaindb.common.utils import serialize
+from .memoize import memoize_from_dict, memoize_to_dict
+
+
+UnspentOutput = namedtuple(
+    'UnspentOutput', (
+        # TODO 'utxo_hash': sha3_256(f'{txid}{output_index}'.encode())
+        # 'utxo_hash',   # noqa
+        'transaction_id',
+        'output_index',
+        'amount',
+        'asset_id',
+        'condition_uri',
+    )
+)
 
 
 class Input(object):
@@ -48,7 +78,7 @@ class Input(object):
         if fulfills is not None and not isinstance(fulfills, TransactionLink):
             raise TypeError('`fulfills` must be a TransactionLink instance')
         if not isinstance(owners_before, list):
-            raise TypeError('`owners_after` must be a list instance')
+            raise TypeError('`owners_before` must be a list instance')
 
         self.fulfillment = fulfillment
         self.fulfills = fulfills
@@ -58,6 +88,11 @@ class Input(object):
         # TODO: If `other !== Fulfillment` return `False`
         return self.to_dict() == other.to_dict()
 
+    # NOTE: This function is used to provide a unique key for a given
+    # Input to suppliment memoization
+    def __hash__(self):
+        return hash((self.fulfillment, self.fulfills))
+
     def to_dict(self):
         """Transforms the object to a Python dictionary.
 
@@ -70,7 +105,7 @@ class Input(object):
         """
         try:
             fulfillment = self.fulfillment.serialize_uri()
-        except (TypeError, AttributeError, ASN1EncodeError):
+        except (TypeError, AttributeError, ASN1EncodeError, ASN1DecodeError):
             fulfillment = _fulfillment_to_details(self.fulfillment)
 
         try:
@@ -112,7 +147,7 @@ class Input(object):
                 InvalidSignature: If an Input's URI couldn't be parsed.
         """
         fulfillment = data['fulfillment']
-        if not isinstance(fulfillment, Fulfillment):
+        if not isinstance(fulfillment, (Fulfillment, type(None))):
             try:
                 fulfillment = Fulfillment.from_uri(data['fulfillment'])
             except ASN1DecodeError:
@@ -128,8 +163,7 @@ class Input(object):
 
 
 def _fulfillment_to_details(fulfillment):
-    """
-    Encode a fulfillment as a details dictionary
+    """Encode a fulfillment as a details dictionary
 
     Args:
         fulfillment: Crypto-conditions Fulfillment object
@@ -138,7 +172,7 @@ def _fulfillment_to_details(fulfillment):
     if fulfillment.type_name == 'ed25519-sha-256':
         return {
             'type': 'ed25519-sha-256',
-            'public_key': base58.b58encode(fulfillment.public_key),
+            'public_key': base58.b58encode(fulfillment.public_key).decode(),
         }
 
     if fulfillment.type_name == 'threshold-sha-256':
@@ -155,26 +189,25 @@ def _fulfillment_to_details(fulfillment):
     raise UnsupportedTypeError(fulfillment.type_name)
 
 
-def _fulfillment_from_details(data):
-    """
-    Load a fulfillment for a signing spec dictionary
+def _fulfillment_from_details(data, _depth=0):
+    """Load a fulfillment for a signing spec dictionary
 
     Args:
         data: tx.output[].condition.details dictionary
     """
+    if _depth == 100:
+        raise ThresholdTooDeep()
+
     if data['type'] == 'ed25519-sha-256':
         public_key = base58.b58decode(data['public_key'])
         return Ed25519Sha256(public_key=public_key)
 
     if data['type'] == 'threshold-sha-256':
-        try:
-            threshold = ThresholdSha256(data['threshold'])
-            for cond in data['subconditions']:
-                cond = _fulfillment_from_details(cond)
-                threshold.add_subfulfillment(cond)
-            return threshold
-        except RecursionError:
-            raise ThresholdTooDeep()
+        threshold = ThresholdSha256(data['threshold'])
+        for cond in data['subconditions']:
+            cond = _fulfillment_from_details(cond, _depth+1)
+            threshold.add_subfulfillment(cond)
+        return threshold
 
     raise UnsupportedTypeError(data.get('type'))
 
@@ -463,22 +496,22 @@ class Transaction(object):
                 spend.
             outputs (:obj:`list` of :class:`~bigchaindb.common.
                 transaction.Output`, optional): Define the assets to lock.
-            asset (dict): Asset payload for this Transaction. ``CREATE`` and
-                ``GENESIS`` Transactions require a dict with a ``data``
+            asset (dict): Asset payload for this Transaction. ``CREATE``
+                Transactions require a dict with a ``data``
                 property while ``TRANSFER`` Transactions require a dict with a
                 ``id`` property.
             metadata (dict):
                 Metadata to be stored along with the Transaction.
             version (string): Defines the version number of a Transaction.
     """
+
     CREATE = 'CREATE'
     TRANSFER = 'TRANSFER'
-    GENESIS = 'GENESIS'
-    ALLOWED_OPERATIONS = (CREATE, TRANSFER, GENESIS)
-    VERSION = '1.0'
+    ALLOWED_OPERATIONS = (CREATE, TRANSFER)
+    VERSION = '2.0'
 
     def __init__(self, operation, asset, inputs=None, outputs=None,
-                 metadata=None, version=None):
+                 metadata=None, version=None, hash_id=None, tx_dict=None):
         """The constructor allows to create a customizable Transaction.
 
             Note:
@@ -496,23 +529,24 @@ class Transaction(object):
                 metadata (dict): Metadata to be stored along with the
                     Transaction.
                 version (string): Defines the version number of a Transaction.
+                hash_id (string): Hash id of the transaction.
         """
-        if operation not in Transaction.ALLOWED_OPERATIONS:
+        if operation not in self.ALLOWED_OPERATIONS:
             allowed_ops = ', '.join(self.__class__.ALLOWED_OPERATIONS)
             raise ValueError('`operation` must be one of {}'
                              .format(allowed_ops))
 
-        # Asset payloads for 'CREATE' and 'GENESIS' operations must be None or
+        # Asset payloads for 'CREATE' operations must be None or
         # dicts holding a `data` property. Asset payloads for 'TRANSFER'
         # operations must be dicts holding an `id` property.
-        if (operation in [Transaction.CREATE, Transaction.GENESIS] and
+        if (operation == self.CREATE and
                 asset is not None and not (isinstance(asset, dict) and 'data' in asset)):
             raise TypeError(('`asset` must be None or a dict holding a `data` '
                              " property instance for '{}' Transactions".format(operation)))
-        elif (operation == Transaction.TRANSFER and
+        elif (operation == self.TRANSFER and
                 not (isinstance(asset, dict) and 'id' in asset)):
             raise TypeError(('`asset` must be a dict holding an `id` property '
-                             "for 'TRANSFER' Transactions".format(operation)))
+                             'for \'TRANSFER\' Transactions'))
 
         if outputs and not isinstance(outputs, list):
             raise TypeError('`outputs` must be a list instance or None')
@@ -529,6 +563,76 @@ class Transaction(object):
         self.inputs = inputs or []
         self.outputs = outputs or []
         self.metadata = metadata
+        self._id = hash_id
+        self.tx_dict = tx_dict
+
+    @property
+    def unspent_outputs(self):
+        """UnspentOutput: The outputs of this transaction, in a data
+        structure containing relevant information for storing them in
+        a UTXO set, and performing validation.
+        """
+        if self.operation == self.CREATE:
+            self._asset_id = self._id
+        elif self.operation == self.TRANSFER:
+            self._asset_id = self.asset['id']
+        return (UnspentOutput(
+            transaction_id=self._id,
+            output_index=output_index,
+            amount=output.amount,
+            asset_id=self._asset_id,
+            condition_uri=output.fulfillment.condition_uri,
+        ) for output_index, output in enumerate(self.outputs))
+
+    @property
+    def spent_outputs(self):
+        """Tuple of :obj:`dict`: Inputs of this transaction. Each input
+        is represented as a dictionary containing a transaction id and
+        output index.
+        """
+        return (
+            input_.fulfills.to_dict()
+            for input_ in self.inputs if input_.fulfills
+        )
+
+    @property
+    def serialized(self):
+        return Transaction._to_str(self.to_dict())
+
+    def _hash(self):
+        self._id = hash_data(self.serialized)
+
+    @classmethod
+    def validate_create(cls, tx_signers, recipients, asset, metadata):
+        if not isinstance(tx_signers, list):
+            raise TypeError('`tx_signers` must be a list instance')
+        if not isinstance(recipients, list):
+            raise TypeError('`recipients` must be a list instance')
+        if len(tx_signers) == 0:
+            raise ValueError('`tx_signers` list cannot be empty')
+        if len(recipients) == 0:
+            raise ValueError('`recipients` list cannot be empty')
+        if not (asset is None or isinstance(asset, dict)):
+            raise TypeError('`asset` must be a dict or None')
+        if not (metadata is None or isinstance(metadata, dict)):
+            raise TypeError('`metadata` must be a dict or None')
+
+        inputs = []
+        outputs = []
+
+        # generate_outputs
+        for recipient in recipients:
+            if not isinstance(recipient, tuple) or len(recipient) != 2:
+                raise ValueError(('Each `recipient` in the list must be a'
+                                  ' tuple of `([<list of public keys>],'
+                                  ' <amount>)`'))
+            pub_keys, amount = recipient
+            outputs.append(Output.generate(pub_keys, amount))
+
+        # generate inputs
+        inputs.append(Input.generate(tx_signers))
+
+        return (inputs, outputs)
 
     @classmethod
     def create(cls, tx_signers, recipients, metadata=None, asset=None):
@@ -558,21 +662,22 @@ class Transaction(object):
             Returns:
                 :class:`~bigchaindb.common.transaction.Transaction`
         """
-        if not isinstance(tx_signers, list):
-            raise TypeError('`tx_signers` must be a list instance')
+
+        (inputs, outputs) = cls.validate_create(tx_signers, recipients, asset, metadata)
+        return cls(cls.CREATE, {'data': asset}, inputs, outputs, metadata)
+
+    @classmethod
+    def validate_transfer(cls, inputs, recipients, asset_id, metadata):
+        if not isinstance(inputs, list):
+            raise TypeError('`inputs` must be a list instance')
+        if len(inputs) == 0:
+            raise ValueError('`inputs` must contain at least one item')
         if not isinstance(recipients, list):
             raise TypeError('`recipients` must be a list instance')
-        if len(tx_signers) == 0:
-            raise ValueError('`tx_signers` list cannot be empty')
         if len(recipients) == 0:
             raise ValueError('`recipients` list cannot be empty')
-        if not (asset is None or isinstance(asset, dict)):
-            raise TypeError('`asset` must be a dict or None')
 
-        inputs = []
         outputs = []
-
-        # generate_outputs
         for recipient in recipients:
             if not isinstance(recipient, tuple) or len(recipient) != 2:
                 raise ValueError(('Each `recipient` in the list must be a'
@@ -581,10 +686,10 @@ class Transaction(object):
             pub_keys, amount = recipient
             outputs.append(Output.generate(pub_keys, amount))
 
-        # generate inputs
-        inputs.append(Input.generate(tx_signers))
+        if not isinstance(asset_id, str):
+            raise TypeError('`asset_id` must be a string')
 
-        return cls(cls.CREATE, {'data': asset}, inputs, outputs, metadata)
+        return (deepcopy(inputs), outputs)
 
     @classmethod
     def transfer(cls, inputs, recipients, asset_id, metadata=None):
@@ -625,27 +730,7 @@ class Transaction(object):
             Returns:
                 :class:`~bigchaindb.common.transaction.Transaction`
         """
-        if not isinstance(inputs, list):
-            raise TypeError('`inputs` must be a list instance')
-        if len(inputs) == 0:
-            raise ValueError('`inputs` must contain at least one item')
-        if not isinstance(recipients, list):
-            raise TypeError('`recipients` must be a list instance')
-        if len(recipients) == 0:
-            raise ValueError('`recipients` list cannot be empty')
-
-        outputs = []
-        for recipient in recipients:
-            if not isinstance(recipient, tuple) or len(recipient) != 2:
-                raise ValueError(('Each `recipient` in the list must be a'
-                                  ' <amount>)`'))
-            pub_keys, amount = recipient
-            outputs.append(Output.generate(pub_keys, amount))
-
-        if not isinstance(asset_id, str):
-            raise TypeError('`asset_id` must be a string')
-
-        inputs = deepcopy(inputs)
+        (inputs, outputs) = cls.validate_transfer(inputs, recipients, asset_id, metadata)
         return cls(cls.TRANSFER, {'id': asset_id}, inputs, outputs, metadata)
 
     def __eq__(self, other):
@@ -756,6 +841,9 @@ class Transaction(object):
         tx_serialized = Transaction._to_str(tx_dict)
         for i, input_ in enumerate(self.inputs):
             self.inputs[i] = self._sign_input(input_, tx_serialized, key_pairs)
+
+        self._hash()
+
         return self
 
     @classmethod
@@ -781,8 +869,9 @@ class Transaction(object):
             return cls._sign_threshold_signature_fulfillment(input_, message,
                                                              key_pairs)
         else:
-            raise ValueError("Fulfillment couldn't be matched to "
-                             'Cryptocondition fulfillment type.')
+            raise ValueError(
+                'Fulfillment couldn\'t be matched to '
+                'Cryptocondition fulfillment type.')
 
     @classmethod
     def _sign_simple_signature_fulfillment(cls, input_, message, key_pairs):
@@ -800,13 +889,16 @@ class Transaction(object):
         #       this should never happen, but then again, never say never.
         input_ = deepcopy(input_)
         public_key = input_.owners_before[0]
+        message = sha3_256(message.encode())
+        if input_.fulfills:
+            message.update('{}{}'.format(
+                input_.fulfills.txid, input_.fulfills.output).encode())
+
         try:
             # cryptoconditions makes no assumptions of the encoding of the
             # message to sign or verify. It only accepts bytestrings
             input_.fulfillment.sign(
-                message.encode(),
-                base58.b58decode(key_pairs[public_key].encode()),
-            )
+                message.digest(), base58.b58decode(key_pairs[public_key].encode()))
         except KeyError:
             raise KeypairMismatchException('Public key {} is not a pair to '
                                            'any of the private keys'
@@ -824,6 +916,11 @@ class Transaction(object):
                 key_pairs (dict): The keys to sign the Transaction with.
         """
         input_ = deepcopy(input_)
+        message = sha3_256(message.encode())
+        if input_.fulfills:
+            message.update('{}{}'.format(
+                input_.fulfills.txid, input_.fulfills.output).encode())
+
         for owner_before in set(input_.owners_before):
             # TODO: CC should throw a KeypairMismatchException, instead of
             #       our manual mapping here
@@ -851,7 +948,8 @@ class Transaction(object):
             # cryptoconditions makes no assumptions of the encoding of the
             # message to sign or verify. It only accepts bytestrings
             for subffill in subffills:
-                subffill.sign(message.encode(), base58.b58decode(private_key.encode()))
+                subffill.sign(
+                    message.digest(), base58.b58decode(private_key.encode()))
         return input_
 
     def inputs_valid(self, outputs=None):
@@ -859,7 +957,7 @@ class Transaction(object):
         Outputs.
 
             Note:
-                Given a `CREATE` or `GENESIS` Transaction is passed,
+                Given a `CREATE` Transaction is passed,
                 dummy values for Outputs are submitted for validation that
                 evaluate parts of the validation-checks to `True`.
 
@@ -871,14 +969,14 @@ class Transaction(object):
             Returns:
                 bool: If all Inputs are valid.
         """
-        if self.operation in (Transaction.CREATE, Transaction.GENESIS):
+        if self.operation == self.CREATE:
             # NOTE: Since in the case of a `CREATE`-transaction we do not have
             #       to check for outputs, we're just submitting dummy
             #       values to the actual method. This simplifies it's logic
             #       greatly, as we do not have to check against `None` values.
             return self._inputs_valid(['dummyvalue'
                                        for _ in self.inputs])
-        elif self.operation == Transaction.TRANSFER:
+        elif self.operation == self.TRANSFER:
             return self._inputs_valid([output.fulfillment.condition_uri
                                        for output in outputs])
         else:
@@ -905,32 +1003,32 @@ class Transaction(object):
             raise ValueError('Inputs and '
                              'output_condition_uris must have the same count')
 
-        tx_dict = self.to_dict()
+        tx_dict = self.tx_dict if self.tx_dict else self.to_dict()
         tx_dict = Transaction._remove_signatures(tx_dict)
+        tx_dict['id'] = None
         tx_serialized = Transaction._to_str(tx_dict)
 
         def validate(i, output_condition_uri=None):
-            """ Validate input against output condition URI """
+            """Validate input against output condition URI"""
             return self._input_valid(self.inputs[i], self.operation,
                                      tx_serialized, output_condition_uri)
 
         return all(validate(i, cond)
                    for i, cond in enumerate(output_condition_uris))
 
-    @staticmethod
-    def _input_valid(input_, operation, tx_serialized, output_condition_uri=None):
+    @lru_cache(maxsize=16384)
+    def _input_valid(self, input_, operation, message, output_condition_uri=None):
         """Validates a single Input against a single Output.
 
             Note:
-                In case of a `CREATE` or `GENESIS` Transaction, this method
+                In case of a `CREATE` Transaction, this method
                 does not validate against `output_condition_uri`.
 
             Args:
                 input_ (:class:`~bigchaindb.common.transaction.
                     Input`) The Input to be signed.
                 operation (str): The type of Transaction.
-                tx_serialized (str): The Transaction used as a message when
-                    initially signing it.
+                message (str): The fulfillment message.
                 output_condition_uri (str, optional): An Output to check the
                     Input against.
 
@@ -944,43 +1042,47 @@ class Transaction(object):
                 ParsingError, ASN1DecodeError, ASN1EncodeError):
             return False
 
-        if operation in (Transaction.CREATE, Transaction.GENESIS):
-            # NOTE: In the case of a `CREATE` or `GENESIS` transaction, the
+        if operation == self.CREATE:
+            # NOTE: In the case of a `CREATE` transaction, the
             #       output is always valid.
             output_valid = True
         else:
             output_valid = output_condition_uri == ccffill.condition_uri
 
+        message = sha3_256(message.encode())
+        if input_.fulfills:
+            message.update('{}{}'.format(
+                input_.fulfills.txid, input_.fulfills.output).encode())
+
         # NOTE: We pass a timestamp to `.validate`, as in case of a timeout
         #       condition we'll have to validate against it
 
         # cryptoconditions makes no assumptions of the encoding of the
         # message to sign or verify. It only accepts bytestrings
-        ffill_valid = parsed_ffill.validate(message=tx_serialized.encode())
+        ffill_valid = parsed_ffill.validate(message=message.digest())
         return output_valid and ffill_valid
 
+    # This function is required by `lru_cache` to create a key for memoization
+    def __hash__(self):
+        return hash(self.id)
+
+    @memoize_to_dict
     def to_dict(self):
         """Transforms the object to a Python dictionary.
 
             Returns:
                 dict: The Transaction as an alternative serialization format.
         """
-        tx = {
+        return {
             'inputs': [input_.to_dict() for input_ in self.inputs],
             'outputs': [output.to_dict() for output in self.outputs],
             'operation': str(self.operation),
             'metadata': self.metadata,
             'asset': self.asset,
             'version': self.version,
+            'id': self._id,
         }
 
-        tx_no_signatures = Transaction._remove_signatures(tx)
-        tx_serialized = Transaction._to_str(tx_no_signatures)
-        tx_id = Transaction._to_hash(tx_serialized)
-
-        tx['id'] = tx_id
-        return tx
-
     @staticmethod
     # TODO: Remove `_dict` prefix of variable.
     def _remove_signatures(tx_dict):
@@ -1010,7 +1112,7 @@ class Transaction(object):
 
     @property
     def id(self):
-        return self.to_hash()
+        return self._id
 
     def to_hash(self):
         return self.to_dict()['id']
@@ -1024,8 +1126,8 @@ class Transaction(object):
         tx = Transaction._remove_signatures(self.to_dict())
         return Transaction._to_str(tx)
 
-    @staticmethod
-    def get_asset_id(transactions):
+    @classmethod
+    def get_asset_id(cls, transactions):
         """Get the asset id from a list of :class:`~.Transactions`.
 
         This is useful when we want to check if the multiple inputs of a
@@ -1049,7 +1151,7 @@ class Transaction(object):
             transactions = [transactions]
 
         # create a set of the transactions' asset ids
-        asset_ids = {tx.id if tx.operation == Transaction.CREATE
+        asset_ids = {tx.id if tx.operation == tx.CREATE
                      else tx.asset['id']
                      for tx in transactions}
 
@@ -1067,14 +1169,17 @@ class Transaction(object):
                 tx_body (dict): The Transaction to be transformed.
         """
         # NOTE: Remove reference to avoid side effects
-        tx_body = deepcopy(tx_body)
+        # tx_body = deepcopy(tx_body)
+        tx_body = rapidjson.loads(rapidjson.dumps(tx_body))
+
         try:
-            proposed_tx_id = tx_body.pop('id')
+            proposed_tx_id = tx_body['id']
         except KeyError:
             raise InvalidHash('No transaction id found!')
 
-        tx_body_no_signatures = Transaction._remove_signatures(tx_body)
-        tx_body_serialized = Transaction._to_str(tx_body_no_signatures)
+        tx_body['id'] = None
+
+        tx_body_serialized = Transaction._to_str(tx_body)
         valid_tx_id = Transaction._to_hash(tx_body_serialized)
 
         if proposed_tx_id != valid_tx_id:
@@ -1083,7 +1188,8 @@ class Transaction(object):
             raise InvalidHash(err_msg.format(proposed_tx_id))
 
     @classmethod
-    def from_dict(cls, tx):
+    @memoize_from_dict
+    def from_dict(cls, tx, skip_schema_validation=True):
         """Transforms a Python dictionary to a Transaction object.
 
             Args:
@@ -1092,8 +1198,133 @@ class Transaction(object):
             Returns:
                 :class:`~bigchaindb.common.transaction.Transaction`
         """
-        cls.validate_id(tx)
+        operation = tx.get('operation', Transaction.CREATE) if isinstance(tx, dict) else Transaction.CREATE
+        cls = Transaction.resolve_class(operation)
+
+        if not skip_schema_validation:
+            cls.validate_id(tx)
+            cls.validate_schema(tx)
+
         inputs = [Input.from_dict(input_) for input_ in tx['inputs']]
         outputs = [Output.from_dict(output) for output in tx['outputs']]
         return cls(tx['operation'], tx['asset'], inputs, outputs,
-                   tx['metadata'], tx['version'])
+                   tx['metadata'], tx['version'], hash_id=tx['id'], tx_dict=tx)
+
+    @classmethod
+    def from_db(cls, bigchain, tx_dict_list):
+        """Helper method that reconstructs a transaction dict that was returned
+        from the database. It checks what asset_id to retrieve, retrieves the
+        asset from the asset table and reconstructs the transaction.
+
+        Args:
+            bigchain (:class:`~bigchaindb.tendermint.BigchainDB`): An instance
+                of BigchainDB used to perform database queries.
+            tx_dict_list (:list:`dict` or :obj:`dict`): The transaction dict or
+                list of transaction dict as returned from the database.
+
+        Returns:
+            :class:`~Transaction`
+
+        """
+        return_list = True
+        if isinstance(tx_dict_list, dict):
+            tx_dict_list = [tx_dict_list]
+            return_list = False
+
+        tx_map = {}
+        tx_ids = []
+        for tx in tx_dict_list:
+            tx.update({'metadata': None})
+            tx_map[tx['id']] = tx
+            tx_ids.append(tx['id'])
+
+        assets = list(bigchain.get_assets(tx_ids))
+        for asset in assets:
+            if asset is not None:
+                tx = tx_map[asset['id']]
+                del asset['id']
+                tx['asset'] = asset
+
+        tx_ids = list(tx_map.keys())
+        metadata_list = list(bigchain.get_metadata(tx_ids))
+        for metadata in metadata_list:
+            tx = tx_map[metadata['id']]
+            tx.update({'metadata': metadata.get('metadata')})
+
+        if return_list:
+            tx_list = []
+            for tx_id, tx in tx_map.items():
+                tx_list.append(cls.from_dict(tx))
+            return tx_list
+        else:
+            tx = list(tx_map.values())[0]
+            return cls.from_dict(tx)
+
+    type_registry = {}
+
+    @staticmethod
+    def register_type(tx_type, tx_class):
+        Transaction.type_registry[tx_type] = tx_class
+
+    def resolve_class(operation):
+        """For the given `tx` based on the `operation` key return its implementation class"""
+
+        create_txn_class = Transaction.type_registry.get(Transaction.CREATE)
+        return Transaction.type_registry.get(operation, create_txn_class)
+
+    @classmethod
+    def validate_schema(cls, tx):
+        pass
+
+    def validate_transfer_inputs(self, bigchain, current_transactions=[]):
+        # store the inputs so that we can check if the asset ids match
+        input_txs = []
+        input_conditions = []
+        for input_ in self.inputs:
+            input_txid = input_.fulfills.txid
+            input_tx = bigchain.get_transaction(input_txid)
+
+            if input_tx is None:
+                for ctxn in current_transactions:
+                    if ctxn.id == input_txid:
+                        input_tx = ctxn
+
+            if input_tx is None:
+                raise InputDoesNotExist("input `{}` doesn't exist"
+                                        .format(input_txid))
+
+            spent = bigchain.get_spent(input_txid, input_.fulfills.output,
+                                       current_transactions)
+            if spent:
+                raise DoubleSpend('input `{}` was already spent'
+                                  .format(input_txid))
+
+            output = input_tx.outputs[input_.fulfills.output]
+            input_conditions.append(output)
+            input_txs.append(input_tx)
+
+        # Validate that all inputs are distinct
+        links = [i.fulfills.to_uri() for i in self.inputs]
+        if len(links) != len(set(links)):
+            raise DoubleSpend('tx "{}" spends inputs twice'.format(self.id))
+
+        # validate asset id
+        asset_id = self.get_asset_id(input_txs)
+        if asset_id != self.asset['id']:
+            raise AssetIdMismatch(('The asset id of the input does not'
+                                   ' match the asset id of the'
+                                   ' transaction'))
+
+        input_amount = sum([input_condition.amount for input_condition in input_conditions])
+        output_amount = sum([output_condition.amount for output_condition in self.outputs])
+
+        if output_amount != input_amount:
+            raise AmountError(('The amount used in the inputs `{}`'
+                               ' needs to be same as the amount used'
+                               ' in the outputs `{}`')
+                              .format(input_amount, output_amount))
+
+        if not self.inputs_valid(input_conditions):
+            raise InvalidSignature('Transaction signature is invalid.')
+
+        return True

bigchaindb/common/transaction_mode_types.py

@@ -0,0 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+BROADCAST_TX_COMMIT = 'broadcast_tx_commit'
+BROADCAST_TX_ASYNC = 'broadcast_tx_async'
+BROADCAST_TX_SYNC = 'broadcast_tx_sync'

bigchaindb/common/utils.py

@@ -1,7 +1,15 @@
-import time
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
 
+import time
+import re
 import rapidjson
 
+import bigchaindb
+from bigchaindb.common.exceptions import ValidationError
+
 
 def gen_timestamp():
     """The Unix time, rounded to the nearest second.
@@ -46,3 +54,112 @@ def deserialize(data):
             string.
     """
     return rapidjson.loads(data)
+
+
+def validate_txn_obj(obj_name, obj, key, validation_fun):
+    """Validate value of `key` in `obj` using `validation_fun`.
+
+        Args:
+            obj_name (str): name for `obj` being validated.
+            obj (dict): dictionary object.
+            key (str): key to be validated in `obj`.
+            validation_fun (function): function used to validate the value
+            of `key`.
+
+        Returns:
+            None: indicates validation successful
+
+        Raises:
+            ValidationError: `validation_fun` will raise exception on failure
+    """
+    backend = bigchaindb.config['database']['backend']
+
+    if backend == 'localmongodb':
+        data = obj.get(key, {})
+        if isinstance(data, dict):
+            validate_all_keys_in_obj(obj_name, data, validation_fun)
+        elif isinstance(data, list):
+            validate_all_items_in_list(obj_name, data, validation_fun)
+
+
+def validate_all_items_in_list(obj_name, data, validation_fun):
+    for item in data:
+        if isinstance(item, dict):
+            validate_all_keys_in_obj(obj_name, item, validation_fun)
+        elif isinstance(item, list):
+            validate_all_items_in_list(obj_name, item, validation_fun)
+
+
+def validate_all_keys_in_obj(obj_name, obj, validation_fun):
+    """Validate all (nested) keys in `obj` by using `validation_fun`.
+
+        Args:
+            obj_name (str): name for `obj` being validated.
+            obj (dict): dictionary object.
+            validation_fun (function): function used to validate the value
+            of `key`.
+
+        Returns:
+            None: indicates validation successful
+
+        Raises:
+            ValidationError: `validation_fun` will raise this error on failure
+    """
+    for key, value in obj.items():
+        validation_fun(obj_name, key)
+        if isinstance(value, dict):
+            validate_all_keys_in_obj(obj_name, value, validation_fun)
+        elif isinstance(value, list):
+            validate_all_items_in_list(obj_name, value, validation_fun)
+
+
+def validate_all_values_for_key_in_obj(obj, key, validation_fun):
+    """Validate value for all (nested) occurrence  of `key` in `obj`
+       using `validation_fun`.
+
+        Args:
+            obj (dict): dictionary object.
+            key (str): key whose value is to be validated.
+            validation_fun (function): function used to validate the value
+            of `key`.
+
+        Raises:
+            ValidationError: `validation_fun` will raise this error on failure
+    """
+    for vkey, value in obj.items():
+        if vkey == key:
+            validation_fun(value)
+        elif isinstance(value, dict):
+            validate_all_values_for_key_in_obj(value, key, validation_fun)
+        elif isinstance(value, list):
+            validate_all_values_for_key_in_list(value, key, validation_fun)
+
+
+def validate_all_values_for_key_in_list(input_list, key, validation_fun):
+    for item in input_list:
+        if isinstance(item, dict):
+            validate_all_values_for_key_in_obj(item, key, validation_fun)
+        elif isinstance(item, list):
+            validate_all_values_for_key_in_list(item, key, validation_fun)
+
+
+def validate_key(obj_name, key):
+    """Check if `key` contains ".", "$" or null characters.
+
+       https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names
+
+        Args:
+            obj_name (str): object name to use when raising exception
+            key (str): key to validated
+
+        Returns:
+            None: validation successful
+
+        Raises:
+            ValidationError: will raise exception in case of regex match.
+    """
+    if re.search(r'^[$]|\.|\x00', key):
+        error_str = ('Invalid key name "{}" in {} object. The '
+                     'key name cannot contain characters '
+                     '".", "$" or null characters').format(key, obj_name)
+        raise ValidationError(error_str)

bigchaindb/config_utils.py

@@ -1,3 +1,8 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 """Utils for reading and setting configuration settings.
 
 The value of each BigchainDB Server configuration setting is
@@ -15,7 +20,7 @@ import os
 import copy
 import json
 import logging
-import collections
+import collections.abc
 from functools import lru_cache
 
 from pkg_resources import iter_entry_points, ResolutionError
@@ -24,7 +29,7 @@ from bigchaindb.common import exceptions
 
 import bigchaindb
 
-from bigchaindb.consensus import BaseConsensusRules
+from bigchaindb.validation import BaseValidationRules
 
 # TODO: move this to a proper configuration file for logging
 logging.getLogger('requests').setLevel(logging.WARNING)
@@ -47,7 +52,7 @@ def map_leafs(func, mapping):
             path = []
 
         for key, val in mapping.items():
-            if isinstance(val, collections.Mapping):
+            if isinstance(val, collections.abc.Mapping):
                 _inner(val, path + [key])
             else:
                 mapping[key] = func(val, path=path+[key])
@@ -75,7 +80,7 @@ def update(d, u):
         mapping: An updated version of d (updated by u).
     """
     for k, v in u.items():
-        if isinstance(v, collections.Mapping):
+        if isinstance(v, collections.abc.Mapping):
             r = update(d.get(k, {}), v)
             d[k] = r
         else:
@@ -108,7 +113,7 @@ def file_config(filename=None):
                 'Failed to parse the JSON configuration from `{}`, {}'.format(filename, err)
             )
 
-    logger.info('Configuration loaded from `{}`'.format(filename))
+        logger.info('Configuration loaded from `{}`'.format(filename))
 
     return config
 
@@ -128,6 +133,7 @@ def env_config(config):
 
     def load_from_env(value, path):
         var_name = CONFIG_SEP.join([CONFIG_PREFIX] + list(map(lambda s: s.upper(), path)))
+
         return os.environ.get(var_name, value)
 
     return map_leafs(load_from_env, config)
@@ -135,7 +141,8 @@ def env_config(config):
 
 def update_types(config, reference, list_sep=':'):
     """Return a new configuration where all the values types
-    are aligned with the ones in the default configuration"""
+    are aligned with the ones in the default configuration
+    """
 
     def _coerce(current, value):
         # Coerce a value to the `current` type.
@@ -226,7 +233,8 @@ def is_configured():
 
 def autoconfigure(filename=None, config=None, force=False):
     """Run ``file_config`` and ``env_config`` if the module has not
-    been initialized."""
+    been initialized.
+    """
     if not force and is_configured():
         logger.debug('System already configured, skipping autoconfiguration')
         return
@@ -245,45 +253,56 @@ def autoconfigure(filename=None, config=None, force=False):
 
     # override configuration with env variables
     newconfig = env_config(newconfig)
-
     if config:
         newconfig = update(newconfig, config)
-
     set_config(newconfig)  # sets bigchaindb.config
 
 
 @lru_cache()
-def load_consensus_plugin(name=None):
-    """Find and load the chosen consensus plugin.
+def load_validation_plugin(name=None):
+    """Find and load the chosen validation plugin.
 
     Args:
         name (string): the name of the entry_point, as advertised in the
             setup.py of the providing package.
 
     Returns:
-        an uninstantiated subclass of ``bigchaindb.consensus.AbstractConsensusRules``
+        an uninstantiated subclass of ``bigchaindb.validation.AbstractValidationRules``
     """
     if not name:
-        return BaseConsensusRules
+        return BaseValidationRules
 
-    # TODO: This will return the first plugin with group `bigchaindb.consensus`
+    # TODO: This will return the first plugin with group `bigchaindb.validation`
     #       and name `name` in the active WorkingSet.
     #       We should probably support Requirements specs in the config, e.g.
-    #       consensus_plugin: 'my-plugin-package==0.0.1;default'
+    #       validation_plugin: 'my-plugin-package==0.0.1;default'
     plugin = None
-    for entry_point in iter_entry_points('bigchaindb.consensus', name):
+    for entry_point in iter_entry_points('bigchaindb.validation', name):
         plugin = entry_point.load()
 
     # No matching entry_point found
     if not plugin:
         raise ResolutionError(
-            'No plugin found in group `bigchaindb.consensus` with name `{}`'.
+            'No plugin found in group `bigchaindb.validation` with name `{}`'.
             format(name))
 
     # Is this strictness desireable?
     # It will probably reduce developer headaches in the wild.
-    if not issubclass(plugin, (BaseConsensusRules,)):
+    if not issubclass(plugin, (BaseValidationRules,)):
         raise TypeError('object of type "{}" does not implement `bigchaindb.'
-                        'consensus.BaseConsensusRules`'.format(type(plugin)))
+                        'validation.BaseValidationRules`'.format(type(plugin)))
 
     return plugin
+
+
+def load_events_plugins(names=None):
+    plugins = []
+
+    if names is None:
+        return plugins
+
+    for name in names:
+        for entry_point in iter_entry_points('bigchaindb.events', name):
+            plugins.append((name, entry_point.load()))
+
+    return plugins

bigchaindb/consensus.py

@@ -1,23 +0,0 @@
-from bigchaindb.voting import Voting
-
-
-class BaseConsensusRules():
-    """Base consensus rules for Bigchain.
-
-    A consensus plugin must expose a class inheriting from this one via an entry_point.
-
-    All methods listed below must be implemented.
-
-    """
-    voting = Voting
-
-    @staticmethod
-    def validate_transaction(bigchain, transaction):
-        """See :meth:`bigchaindb.models.Transaction.validate`
-        for documentation."""
-        return transaction.validate(bigchain)
-
-    @staticmethod
-    def validate_block(bigchain, block):
-        """See :meth:`bigchaindb.models.Block.validate` for documentation."""
-        return block.validate(bigchain)

bigchaindb/core.py

@@ -1,648 +1,271 @@
-import random
-import statsd
-from time import time
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
 
-from bigchaindb import exceptions as core_exceptions
-from bigchaindb.common import crypto, exceptions
-from bigchaindb.common.utils import gen_timestamp, serialize
+"""This module contains all the goodness to integrate BigchainDB
+with Tendermint.
+"""
+import logging
+import sys
 
-import bigchaindb
+from abci.application import BaseApplication
+from abci import CodeTypeOk
 
-from bigchaindb import backend, config_utils, fastquery
-from bigchaindb.consensus import BaseConsensusRules
-from bigchaindb.models import Block, Transaction
+from bigchaindb import BigchainDB
+from bigchaindb.elections.election import Election
+from bigchaindb.version import __tm_supported_versions__
+from bigchaindb.utils import tendermint_version_is_compatible
+from bigchaindb.tendermint_utils import (decode_transaction,
+                                         calculate_hash)
+from bigchaindb.lib import Block
+import bigchaindb.upsert_validator.validator_utils as vutils
+from bigchaindb.events import EventTypes, Event
 
 
-class Bigchain(object):
-    """Bigchain API
+CodeTypeError = 1
+logger = logging.getLogger(__name__)
 
-    Create, read, sign, write transactions to the database
+
+class App(BaseApplication):
+    """Bridge between BigchainDB and Tendermint.
+
+    The role of this class is to expose the BigchainDB
+    transaction logic to Tendermint Core.
     """
 
-    BLOCK_INVALID = 'invalid'
-    """return if a block has been voted invalid"""
+    def __init__(self, abci, bigchaindb=None, events_queue=None,):
+        super().__init__(abci)
+        self.events_queue = events_queue
+        self.bigchaindb = bigchaindb or BigchainDB()
+        self.block_txn_ids = []
+        self.block_txn_hash = ''
+        self.block_transactions = []
+        self.validators = None
+        self.new_height = None
+        self.chain = self.bigchaindb.get_latest_abci_chain()
 
-    BLOCK_VALID = TX_VALID = 'valid'
-    """return if a block is valid, or tx is in valid block"""
+    def log_abci_migration_error(self, chain_id, validators):
+        logger.error('An ABCI chain migration is in process. '
+                     'Download the new ABCI client and configure it with '
+                     f'chain_id={chain_id} and validators={validators}.')
 
-    BLOCK_UNDECIDED = TX_UNDECIDED = 'undecided'
-    """return if block is undecided, or tx is in undecided block"""
+    def abort_if_abci_chain_is_not_synced(self):
+        if self.chain is None or self.chain['is_synced']:
+            return
 
-    TX_IN_BACKLOG = 'backlog'
-    """return if transaction is in backlog"""
+        validators = self.bigchaindb.get_validators()
+        self.log_abci_migration_error(self.chain['chain_id'], validators)
+        sys.exit(1)
 
-    def __init__(self, public_key=None, private_key=None, keyring=[], connection=None, backlog_reassign_delay=None):
-        """Initialize the Bigchain instance
+    def init_chain(self, genesis):
+        """Initialize chain upon genesis or a migration"""
 
-        A Bigchain instance has several configuration parameters (e.g. host).
-        If a parameter value is passed as an argument to the Bigchain
-        __init__ method, then that is the value it will have.
-        Otherwise, the parameter value will come from an environment variable.
-        If that environment variable isn't set, then the value
-        will come from the local configuration file. And if that variable
-        isn't in the local configuration file, then the parameter will have
-        its default value (defined in bigchaindb.__init__).
+        app_hash = ''
+        height = 0
 
-        Args:
-            public_key (str): the base58 encoded public key for the ED25519 curve.
-            private_key (str): the base58 encoded private key for the ED25519 curve.
-            keyring (list[str]): list of base58 encoded public keys of the federation nodes.
-            connection (:class:`~bigchaindb.backend.connection.Connection`):
-                A connection to the database.
-        """
+        known_chain = self.bigchaindb.get_latest_abci_chain()
+        if known_chain is not None:
+            chain_id = known_chain['chain_id']
 
-        config_utils.autoconfigure()
+            if known_chain['is_synced']:
+                msg = (f'Got invalid InitChain ABCI request ({genesis}) - '
+                       f'the chain {chain_id} is already synced.')
+                logger.error(msg)
+                sys.exit(1)
 
-        self.me = public_key or bigchaindb.config['keypair']['public']
-        self.me_private = private_key or bigchaindb.config['keypair']['private']
-        self.nodes_except_me = keyring or bigchaindb.config['keyring']
+            if chain_id != genesis.chain_id:
+                validators = self.bigchaindb.get_validators()
+                self.log_abci_migration_error(chain_id, validators)
+                sys.exit(1)
 
-        if backlog_reassign_delay is None:
-            backlog_reassign_delay = bigchaindb.config['backlog_reassign_delay']
-        self.backlog_reassign_delay = backlog_reassign_delay
+            # set migration values for app hash and height
+            block = self.bigchaindb.get_latest_block()
+            app_hash = '' if block is None else block['app_hash']
+            height = 0 if block is None else block['height'] + 1
 
-        consensusPlugin = bigchaindb.config.get('consensus_plugin')
+        known_validators = self.bigchaindb.get_validators()
+        validator_set = [vutils.decode_validator(v)
+                         for v in genesis.validators]
 
-        if consensusPlugin:
-            self.consensus = config_utils.load_consensus_plugin(consensusPlugin)
+        if known_validators and known_validators != validator_set:
+            self.log_abci_migration_error(known_chain['chain_id'],
+                                          known_validators)
+            sys.exit(1)
+
+        block = Block(app_hash=app_hash, height=height, transactions=[])
+        self.bigchaindb.store_block(block._asdict())
+        self.bigchaindb.store_validator_set(height + 1, validator_set)
+        abci_chain_height = 0 if known_chain is None else known_chain['height']
+        self.bigchaindb.store_abci_chain(abci_chain_height,
+                                         genesis.chain_id, True)
+        self.chain = {'height': abci_chain_height, 'is_synced': True,
+                      'chain_id': genesis.chain_id}
+        return self.abci.ResponseInitChain()
+
+    def info(self, request):
+        """Return height of the latest committed block."""
+
+        self.abort_if_abci_chain_is_not_synced()
+
+        # Check if BigchainDB supports the Tendermint version
+        if not (hasattr(request, 'version') and tendermint_version_is_compatible(request.version)):
+            logger.error(f'Unsupported Tendermint version: {getattr(request, "version", "no version")}.'
+                         f' Currently, BigchainDB only supports {__tm_supported_versions__}. Exiting!')
+            sys.exit(1)
+
+        logger.info(f"Tendermint version: {request.version}")
+
+        r = self.abci.ResponseInfo()
+        block = self.bigchaindb.get_latest_block()
+        if block:
+            chain_shift = 0 if self.chain is None else self.chain['height']
+            r.last_block_height = block['height'] - chain_shift
+            r.last_block_app_hash = block['app_hash'].encode('utf-8')
         else:
-            self.consensus = BaseConsensusRules
+            r.last_block_height = 0
+            r.last_block_app_hash = b''
+        return r
 
-        self.connection = connection if connection else backend.connect(**bigchaindb.config['database'])
-        if not self.me or not self.me_private:
-            raise exceptions.KeypairNotFoundException()
-
-        self.statsd = statsd.StatsClient(bigchaindb.config['graphite']['host'])
-
-    federation = property(lambda self: set(self.nodes_except_me + [self.me]))
-    """ Set of federation member public keys """
-
-    def write_transaction(self, signed_transaction):
-        """Write the transaction to bigchain.
-
-        When first writing a transaction to the bigchain the transaction will be kept in a backlog until
-        it has been validated by the nodes of the federation.
+    def check_tx(self, raw_transaction):
+        """Validate the transaction before entry into
+        the mempool.
 
         Args:
-            signed_transaction (Transaction): transaction with the `signature` included.
-
-        Returns:
-            dict: database response
+            raw_tx: a raw string (in bytes) transaction.
         """
-        signed_transaction = signed_transaction.to_dict()
 
-        # we will assign this transaction to `one` node. This way we make sure that there are no duplicate
-        # transactions on the bigchain
-        if self.nodes_except_me:
-            assignee = random.choice(self.nodes_except_me)
+        self.abort_if_abci_chain_is_not_synced()
+
+        logger.debug('check_tx: %s', raw_transaction)
+        transaction = decode_transaction(raw_transaction)
+        if self.bigchaindb.is_valid_transaction(transaction):
+            logger.debug('check_tx: VALID')
+            return self.abci.ResponseCheckTx(code=CodeTypeOk)
         else:
-            # I am the only node
-            assignee = self.me
+            logger.debug('check_tx: INVALID')
+            return self.abci.ResponseCheckTx(code=CodeTypeError)
 
-        signed_transaction.update({'assignee': assignee})
-        signed_transaction.update({'assignment_timestamp': time()})
+    def begin_block(self, req_begin_block):
+        """Initialize list of transaction.
+        Args:
+            req_begin_block: block object which contains block header
+            and block hash.
+        """
+        self.abort_if_abci_chain_is_not_synced()
 
-        # write to the backlog
-        return backend.query.write_transaction(self.connection, signed_transaction)
+        chain_shift = 0 if self.chain is None else self.chain['height']
+        logger.debug('BEGIN BLOCK, height:%s, num_txs:%s',
+                     req_begin_block.header.height + chain_shift,
+                     req_begin_block.header.num_txs)
 
-    def reassign_transaction(self, transaction):
-        """Assign a transaction to a new node
+        self.block_txn_ids = []
+        self.block_transactions = []
+        return self.abci.ResponseBeginBlock()
+
+    def deliver_tx(self, raw_transaction):
+        """Validate the transaction before mutating the state.
 
         Args:
-            transaction (dict): assigned transaction
-
-        Returns:
-            dict: database response or None if no reassignment is possible
+            raw_tx: a raw string (in bytes) transaction.
         """
 
-        other_nodes = tuple(
-            self.federation.difference([transaction['assignee']])
-        )
-        new_assignee = random.choice(other_nodes) if other_nodes else self.me
+        self.abort_if_abci_chain_is_not_synced()
 
-        return backend.query.update_transaction(
-                self.connection, transaction['id'],
-                {'assignee': new_assignee, 'assignment_timestamp': time()})
+        logger.debug('deliver_tx: %s', raw_transaction)
+        transaction = self.bigchaindb.is_valid_transaction(
+            decode_transaction(raw_transaction), self.block_transactions)
 
-    def delete_transaction(self, *transaction_id):
-        """Delete a transaction from the backlog.
-
-        Args:
-            *transaction_id (str): the transaction(s) to delete
-
-        Returns:
-            The database response.
-        """
-
-        return backend.query.delete_transaction(self.connection, *transaction_id)
-
-    def get_stale_transactions(self):
-        """Get a cursor of stale transactions.
-
-        Transactions are considered stale if they have been assigned a node, but are still in the
-        backlog after some amount of time specified in the configuration
-        """
-
-        return backend.query.get_stale_transactions(self.connection, self.backlog_reassign_delay)
-
-    def validate_transaction(self, transaction):
-        """Validate a transaction.
-
-        Args:
-            transaction (Transaction): transaction to validate.
-
-        Returns:
-            The transaction if the transaction is valid else it raises an
-            exception describing the reason why the transaction is invalid.
-        """
-
-        return self.consensus.validate_transaction(self, transaction)
-
-    def is_new_transaction(self, txid, exclude_block_id=None):
-        """
-        Return True if the transaction does not exist in any
-        VALID or UNDECIDED block. Return False otherwise.
-
-        Args:
-            txid (str): Transaction ID
-            exclude_block_id (str): Exclude block from search
-        """
-        block_statuses = self.get_blocks_status_containing_tx(txid)
-        block_statuses.pop(exclude_block_id, None)
-        for status in block_statuses.values():
-            if status != self.BLOCK_INVALID:
-                return False
-        return True
-
-    def get_block(self, block_id, include_status=False):
-        """Get the block with the specified `block_id` (and optionally its status)
-
-        Returns the block corresponding to `block_id` or None if no match is
-        found.
-
-        Args:
-            block_id (str): transaction id of the transaction to get
-            include_status (bool): also return the status of the block
-                       the return value is then a tuple: (block, status)
-        """
-        # get block from database
-        block_dict = backend.query.get_block(self.connection, block_id)
-        # get the asset ids from the block
-        if block_dict:
-            asset_ids = Block.get_asset_ids(block_dict)
-            # get the assets from the database
-            assets = self.get_assets(asset_ids)
-            # add the assets to the block transactions
-            block_dict = Block.couple_assets(block_dict, assets)
-
-        status = None
-        if include_status:
-            if block_dict:
-                status = self.block_election_status(block_dict)
-            return block_dict, status
+        if not transaction:
+            logger.debug('deliver_tx: INVALID')
+            return self.abci.ResponseDeliverTx(code=CodeTypeError)
         else:
-            return block_dict
+            logger.debug('storing tx')
+            self.block_txn_ids.append(transaction.id)
+            self.block_transactions.append(transaction)
+            return self.abci.ResponseDeliverTx(code=CodeTypeOk)
 
-    def get_transaction(self, txid, include_status=False):
-        """Get the transaction with the specified `txid` (and optionally its status)
-
-        This query begins by looking in the bigchain table for all blocks containing
-        a transaction with the specified `txid`. If one of those blocks is valid, it
-        returns the matching transaction from that block. Else if some of those
-        blocks are undecided, it returns a matching transaction from one of them. If
-        the transaction was found in invalid blocks only, or in no blocks, then this
-        query looks for a matching transaction in the backlog table, and if it finds
-        one there, it returns that.
+    def end_block(self, request_end_block):
+        """Calculate block hash using transaction ids and previous block
+        hash to be stored in the next block.
 
         Args:
-            txid (str): transaction id of the transaction to get
-            include_status (bool): also return the status of the transaction
-                                   the return value is then a tuple: (tx, status)
-
-        Returns:
-            A :class:`~.models.Transaction` instance if the transaction
-            was found in a valid block, an undecided block, or the backlog table,
-            otherwise ``None``.
-            If :attr:`include_status` is ``True``, also returns the
-            transaction's status if the transaction was found.
+            height (int): new height of the chain.
         """
 
-        response, tx_status = None, None
+        self.abort_if_abci_chain_is_not_synced()
 
-        validity = self.get_blocks_status_containing_tx(txid)
-        check_backlog = True
+        chain_shift = 0 if self.chain is None else self.chain['height']
 
-        if validity:
-            # Disregard invalid blocks, and return if there are no valid or undecided blocks
-            validity = {_id: status for _id, status in validity.items()
-                        if status != Bigchain.BLOCK_INVALID}
-            if validity:
+        height = request_end_block.height + chain_shift
+        self.new_height = height
 
-                # The transaction _was_ found in an undecided or valid block,
-                # so there's no need to look in the backlog table
-                check_backlog = False
+        # store pre-commit state to recover in case there is a crash during
+        # `end_block` or `commit`
+        logger.debug(f'Updating pre-commit state: {self.new_height}')
+        pre_commit_state = dict(height=self.new_height,
+                                transactions=self.block_txn_ids)
+        self.bigchaindb.store_pre_commit_state(pre_commit_state)
 
-                tx_status = self.TX_UNDECIDED
-                # If the transaction is in a valid or any undecided block, return it. Does not check
-                # if transactions in undecided blocks are consistent, but selects the valid block
-                # before undecided ones
-                for target_block_id in validity:
-                    if validity[target_block_id] == Bigchain.BLOCK_VALID:
-                        tx_status = self.TX_VALID
-                        break
+        block_txn_hash = calculate_hash(self.block_txn_ids)
+        block = self.bigchaindb.get_latest_block()
 
-                # Query the transaction in the target block and return
-                response = backend.query.get_transaction_from_block(self.connection, txid, target_block_id)
-
-        if check_backlog:
-            response = backend.query.get_transaction_from_backlog(self.connection, txid)
-
-            if response:
-                tx_status = self.TX_IN_BACKLOG
-
-        if response:
-            if tx_status == self.TX_IN_BACKLOG:
-                response = Transaction.from_dict(response)
-            else:
-                # If we are reading from the bigchain collection the asset is
-                # not in the transaction so we need to fetch the asset and
-                # reconstruct the transaction.
-                response = Transaction.from_db(self, response)
-
-        if include_status:
-            return response, tx_status
+        if self.block_txn_ids:
+            self.block_txn_hash = calculate_hash([block['app_hash'], block_txn_hash])
         else:
-            return response
+            self.block_txn_hash = block['app_hash']
 
-    def get_status(self, txid):
-        """Retrieve the status of a transaction with `txid` from bigchain.
+        validator_update = Election.process_block(self.bigchaindb,
+                                                  self.new_height,
+                                                  self.block_transactions)
 
-        Args:
-            txid (str): transaction id of the transaction to query
+        return self.abci.ResponseEndBlock(validator_updates=validator_update)
 
-        Returns:
-            (string): transaction status ('valid', 'undecided',
-            or 'backlog'). If no transaction with that `txid` was found it
-            returns `None`
-        """
-        _, status = self.get_transaction(txid, include_status=True)
-        return status
+    def commit(self):
+        """Store the new height and along with block hash."""
 
-    def get_blocks_status_containing_tx(self, txid):
-        """Retrieve block ids and statuses related to a transaction
+        self.abort_if_abci_chain_is_not_synced()
 
-        Transactions may occur in multiple blocks, but no more than one valid block.
+        data = self.block_txn_hash.encode('utf-8')
 
-        Args:
-            txid (str): transaction id of the transaction to query
+        # register a new block only when new transactions are received
+        if self.block_txn_ids:
+            self.bigchaindb.store_bulk_transactions(self.block_transactions)
 
-        Returns:
-            A dict of blocks containing the transaction,
-            e.g. {block_id_1: 'valid', block_id_2: 'invalid' ...}, or None
-        """
+        block = Block(app_hash=self.block_txn_hash,
+                      height=self.new_height,
+                      transactions=self.block_txn_ids)
+        # NOTE: storing the block should be the last operation during commit
+        # this effects crash recovery. Refer BEP#8 for details
+        self.bigchaindb.store_block(block._asdict())
 
-        # First, get information on all blocks which contain this transaction
-        blocks = backend.query.get_blocks_status_from_transaction(self.connection, txid)
-        if blocks:
-            # Determine the election status of each block
-            validity = {block['id']: self.block_election_status(block)
-                        for block in blocks}
+        logger.debug('Commit-ing new block with hash: apphash=%s ,'
+                     'height=%s, txn ids=%s', data, self.new_height,
+                     self.block_txn_ids)
 
-            # NOTE: If there are multiple valid blocks with this transaction,
-            # something has gone wrong
-            if list(validity.values()).count(Bigchain.BLOCK_VALID) > 1:
-                block_ids = str([block for block in validity
-                                 if validity[block] == Bigchain.BLOCK_VALID])
-                raise core_exceptions.CriticalDoubleInclusion(
-                    'Transaction {tx} is present in '
-                    'multiple valid blocks: {block_ids}'
-                    .format(tx=txid, block_ids=block_ids))
+        if self.events_queue:
+            event = Event(EventTypes.BLOCK_VALID, {
+                'height': self.new_height,
+                'transactions': self.block_transactions
+            })
+            self.events_queue.put(event)
 
-            return validity
+        return self.abci.ResponseCommit(data=data)
 
-        else:
-            return None
 
-    def get_asset_by_id(self, asset_id):
-        """Returns the asset associated with an asset_id.
+def rollback(b):
+    pre_commit = b.get_pre_commit_state()
 
-            Args:
-                asset_id (str): The asset id.
+    if pre_commit is None:
+        # the pre_commit record is first stored in the first `end_block`
+        return
 
-            Returns:
-                dict if the asset exists else None.
-        """
-        cursor = backend.query.get_asset_by_id(self.connection, asset_id)
-        cursor = list(cursor)
-        if cursor:
-            return cursor[0]['asset']
+    latest_block = b.get_latest_block()
+    if latest_block is None:
+        logger.error('Found precommit state but no blocks!')
+        sys.exit(1)
 
-    def get_spent(self, txid, output):
-        """Check if a `txid` was already used as an input.
-
-        A transaction can be used as an input for another transaction. Bigchain
-        needs to make sure that a given `(txid, output)` is only used once.
-
-        This method will check if the `(txid, output)` has already been
-        spent in a transaction that is in either the `VALID`, `UNDECIDED` or
-        `BACKLOG` state.
-
-        Args:
-            txid (str): The id of the transaction
-            output (num): the index of the output in the respective transaction
-
-        Returns:
-            The transaction (Transaction) that used the `(txid, output)` as an
-            input else `None`
-
-        Raises:
-            CriticalDoubleSpend: If the given `(txid, output)` was spent in
-            more than one valid transaction.
-        """
-        # checks if an input was already spent
-        # checks if the bigchain has any transaction with input {'txid': ...,
-        # 'output': ...}
-        transactions = list(backend.query.get_spent(self.connection, txid,
-                                                    output))
-
-        # a transaction_id should have been spent at most one time
-        # determine if these valid transactions appear in more than one valid
-        # block
-        num_valid_transactions = 0
-        non_invalid_transactions = []
-        for transaction in transactions:
-            # ignore transactions in invalid blocks
-            # FIXME: Isn't there a faster solution than doing I/O again?
-            _, status = self.get_transaction(transaction['id'],
-                                             include_status=True)
-            if status == self.TX_VALID:
-                num_valid_transactions += 1
-            # `txid` can only have been spent in at most on valid block.
-            if num_valid_transactions > 1:
-                raise core_exceptions.CriticalDoubleSpend(
-                    '`{}` was spent more than once. There is a problem'
-                    ' with the chain'.format(txid))
-            # if its not and invalid transaction
-            if status is not None:
-                non_invalid_transactions.append(transaction)
-
-        if non_invalid_transactions:
-            return Transaction.from_dict(non_invalid_transactions[0])
-
-        # Either no transaction was returned spending the `(txid, output)` as
-        # input or the returned transactions are not valid.
-
-    def get_owned_ids(self, owner):
-        """Retrieve a list of ``txid`` s that can be used as inputs.
-
-        Args:
-            owner (str): base58 encoded public key.
-
-        Returns:
-            :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s
-            pointing to another transaction's condition
-        """
-        return self.get_outputs_filtered(owner, spent=False)
-
-    @property
-    def fastquery(self):
-        return fastquery.FastQuery(self.connection, self.me)
-
-    def get_outputs_filtered(self, owner, spent=None):
-        """
-        Get a list of output links filtered on some criteria
-
-        Args:
-            owner (str): base58 encoded public_key.
-            spent (bool): If ``True`` return only the spent outputs. If
-                          ``False`` return only unspent outputs. If spent is
-                          not specified (``None``) return all outputs.
-
-        Returns:
-            :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s
-            pointing to another transaction's condition
-        """
-        outputs = self.fastquery.get_outputs_by_public_key(owner)
-        if spent is None:
-            return outputs
-        elif spent is True:
-            return self.fastquery.filter_unspent_outputs(outputs)
-        elif spent is False:
-            return self.fastquery.filter_spent_outputs(outputs)
-
-    def get_transactions_filtered(self, asset_id, operation=None):
-        """
-        Get a list of transactions filtered on some criteria
-        """
-        txids = backend.query.get_txids_filtered(self.connection, asset_id,
-                                                 operation)
-        for txid in txids:
-            tx, status = self.get_transaction(txid, True)
-            if status == self.TX_VALID:
-                yield tx
-
-    def create_block(self, validated_transactions):
-        """Creates a block given a list of `validated_transactions`.
-
-        Note that this method does not validate the transactions. Transactions
-        should be validated before calling create_block.
-
-        Args:
-            validated_transactions (list(Transaction)): list of validated
-                                                        transactions.
-
-        Returns:
-            Block: created block.
-        """
-        # Prevent the creation of empty blocks
-        if not validated_transactions:
-            raise exceptions.OperationError('Empty block creation is not '
-                                            'allowed')
-
-        voters = list(self.federation)
-        block = Block(validated_transactions, self.me, gen_timestamp(), voters)
-        block = block.sign(self.me_private)
-
-        return block
-
-    # TODO: check that the votings structure is correctly constructed
-    def validate_block(self, block):
-        """Validate a block.
-
-        Args:
-            block (Block): block to validate.
-
-        Returns:
-            The block if the block is valid else it raises and exception
-            describing the reason why the block is invalid.
-        """
-        return self.consensus.validate_block(self, block)
-
-    def has_previous_vote(self, block_id):
-        """Check for previous votes from this node
-
-        Args:
-            block_id (str): the id of the block to check
-
-        Returns:
-            bool: :const:`True` if this block already has a
-            valid vote from this node, :const:`False` otherwise.
-
-        """
-        votes = list(backend.query.get_votes_by_block_id_and_voter(self.connection, block_id, self.me))
-        el, _ = self.consensus.voting.partition_eligible_votes(votes, [self.me])
-        return bool(el)
-
-    def write_block(self, block):
-        """Write a block to bigchain.
-
-        Args:
-            block (Block): block to write to bigchain.
-        """
-
-        # Decouple assets from block
-        assets, block_dict = block.decouple_assets()
-        # write the assets
-        if assets:
-            self.write_assets(assets)
-
-        # write the block
-        return backend.query.write_block(self.connection, block_dict)
-
-    def prepare_genesis_block(self):
-        """Prepare a genesis block."""
-
-        metadata = {'message': 'Hello World from the BigchainDB'}
-        transaction = Transaction.create([self.me], [([self.me], 1)],
-                                         metadata=metadata)
-
-        # NOTE: The transaction model doesn't expose an API to generate a
-        #       GENESIS transaction, as this is literally the only usage.
-        transaction.operation = 'GENESIS'
-        transaction = transaction.sign([self.me_private])
-
-        # create the block
-        return self.create_block([transaction])
-
-    def create_genesis_block(self):
-        """Create the genesis block
-
-        Block created when bigchain is first initialized. This method is not atomic, there might be concurrency
-        problems if multiple instances try to write the genesis block when the BigchainDB Federation is started,
-        but it's a highly unlikely scenario.
-        """
-
-        # 1. create one transaction
-        # 2. create the block with one transaction
-        # 3. write the block to the bigchain
-
-        blocks_count = backend.query.count_blocks(self.connection)
-
-        if blocks_count:
-            raise exceptions.GenesisBlockAlreadyExistsError('Cannot create the Genesis block')
-
-        block = self.prepare_genesis_block()
-        self.write_block(block)
-
-        return block
-
-    def vote(self, block_id, previous_block_id, decision, invalid_reason=None):
-        """Create a signed vote for a block given the
-        :attr:`previous_block_id` and the :attr:`decision` (valid/invalid).
-
-        Args:
-            block_id (str): The id of the block to vote on.
-            previous_block_id (str): The id of the previous block.
-            decision (bool): Whether the block is valid or invalid.
-            invalid_reason (Optional[str]): Reason the block is invalid
-        """
-
-        if block_id == previous_block_id:
-            raise exceptions.CyclicBlockchainError()
-
-        vote = {
-            'voting_for_block': block_id,
-            'previous_block': previous_block_id,
-            'is_block_valid': decision,
-            'invalid_reason': invalid_reason,
-            'timestamp': gen_timestamp()
-        }
-
-        vote_data = serialize(vote)
-        signature = crypto.PrivateKey(self.me_private).sign(vote_data.encode())
-
-        vote_signed = {
-            'node_pubkey': self.me,
-            'signature': signature.decode(),
-            'vote': vote
-        }
-
-        return vote_signed
-
-    def write_vote(self, vote):
-        """Write the vote to the database."""
-        return backend.query.write_vote(self.connection, vote)
-
-    def get_last_voted_block(self):
-        """Returns the last block that this node voted on."""
-
-        last_block_id = backend.query.get_last_voted_block_id(self.connection,
-                                                              self.me)
-        return Block.from_dict(self.get_block(last_block_id))
-
-    def block_election(self, block):
-        if type(block) != dict:
-            block = block.to_dict()
-        votes = list(backend.query.get_votes_by_block_id(self.connection,
-                                                         block['id']))
-        return self.consensus.voting.block_election(block, votes,
-                                                    self.federation)
-
-    def block_election_status(self, block):
-        """Tally the votes on a block, and return the status:
-           valid, invalid, or undecided."""
-        return self.block_election(block)['status']
-
-    def get_assets(self, asset_ids):
-        """
-        Return a list of assets that match the asset_ids
-
-        Args:
-            asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to
-                retrieve from the database.
-
-        Returns:
-            list: The list of assets returned from the database.
-        """
-        return backend.query.get_assets(self.connection, asset_ids)
-
-    def write_assets(self, assets):
-        """
-        Writes a list of assets into the database.
-
-        Args:
-            assets (:obj:`list` of :obj:`dict`): A list of assets to write to
-                the database.
-        """
-        return backend.query.write_assets(self.connection, assets)
-
-    def text_search(self, search, *, limit=0):
-        """
-        Return an iterator of assets that match the text search
-
-        Args:
-            search (str): Text search string to query the text index
-            limit (int, optional): Limit the number of returned documents.
-
-        Returns:
-            iter: An iterator of assets that match the text search.
-        """
-        assets = backend.query.text_search(self.connection, search, limit=limit)
-
-        # TODO: This is not efficient. There may be a more efficient way to
-        #       query by storing block ids with the assets and using fastquery.
-        #       See https://github.com/bigchaindb/bigchaindb/issues/1496
-        for asset in assets:
-            tx, status = self.get_transaction(asset['id'], True)
-            if status == self.TX_VALID:
-                yield asset
+    # NOTE: the pre-commit state is always at most 1 block ahead of the commited state
+    if latest_block['height'] < pre_commit['height']:
+        Election.rollback(b, pre_commit['height'], pre_commit['transactions'])
+        b.delete_transactions(pre_commit['transactions'])

bigchaindb/elections/election.py

@@ -0,0 +1,355 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+from collections import OrderedDict
+
+import base58
+from uuid import uuid4
+
+from bigchaindb import backend
+from bigchaindb.elections.vote import Vote
+from bigchaindb.common.exceptions import (InvalidSignature,
+                                          MultipleInputsError,
+                                          InvalidProposer,
+                                          UnequalValidatorSet,
+                                          DuplicateTransaction)
+from bigchaindb.tendermint_utils import key_from_base64, public_key_to_base64
+from bigchaindb.common.crypto import (public_key_from_ed25519_key)
+from bigchaindb.common.transaction import Transaction
+from bigchaindb.common.schema import (_validate_schema,
+                                      TX_SCHEMA_COMMON,
+                                      TX_SCHEMA_CREATE)
+
+
+class Election(Transaction):
+    """Represents election transactions.
+
+       To implement a custom election, create a class deriving from this one
+       with OPERATION set to the election operation, ALLOWED_OPERATIONS
+       set to (OPERATION,), CREATE set to OPERATION.
+    """
+
+    OPERATION = None
+    # Custom validation schema
+    TX_SCHEMA_CUSTOM = None
+    # Election Statuses:
+    ONGOING = 'ongoing'
+    CONCLUDED = 'concluded'
+    INCONCLUSIVE = 'inconclusive'
+    # Vote ratio to approve an election
+    ELECTION_THRESHOLD = 2 / 3
+
+    @classmethod
+    def get_validator_change(cls, bigchain):
+        """Return the validator set from the most recent approved block
+
+        :return: {
+            'height': <block_height>,
+            'validators': <validator_set>
+        }
+        """
+        latest_block = bigchain.get_latest_block()
+        if latest_block is None:
+            return None
+        return bigchain.get_validator_change(latest_block['height'])
+
+    @classmethod
+    def get_validators(cls, bigchain, height=None):
+        """Return a dictionary of validators with key as `public_key` and
+           value as the `voting_power`
+        """
+        validators = {}
+        for validator in bigchain.get_validators(height):
+            # NOTE: we assume that Tendermint encodes public key in base64
+            public_key = public_key_from_ed25519_key(key_from_base64(validator['public_key']['value']))
+            validators[public_key] = validator['voting_power']
+
+        return validators
+
+    @classmethod
+    def recipients(cls, bigchain):
+        """Convert validator dictionary to a recipient list for `Transaction`"""
+
+        recipients = []
+        for public_key, voting_power in cls.get_validators(bigchain).items():
+            recipients.append(([public_key], voting_power))
+
+        return recipients
+
+    @classmethod
+    def is_same_topology(cls, current_topology, election_topology):
+        voters = {}
+        for voter in election_topology:
+            if len(voter.public_keys) > 1:
+                return False
+
+            [public_key] = voter.public_keys
+            voting_power = voter.amount
+            voters[public_key] = voting_power
+
+        # Check whether the voters and their votes is same to that of the
+        # validators and their voting power in the network
+        return current_topology == voters
+
+    def validate(self, bigchain, current_transactions=[]):
+        """Validate election transaction
+
+        NOTE:
+        * A valid election is initiated by an existing validator.
+
+        * A valid election is one where voters are validators and votes are
+          allocated according to the voting power of each validator node.
+
+        Args:
+            :param bigchain: (BigchainDB) an instantiated bigchaindb.lib.BigchainDB object.
+            :param current_transactions: (list) A list of transactions to be validated along with the election
+
+        Returns:
+            Election: a Election object or an object of the derived Election subclass.
+
+        Raises:
+            ValidationError: If the election is invalid
+        """
+        input_conditions = []
+
+        duplicates = any(txn for txn in current_transactions if txn.id == self.id)
+        if bigchain.is_committed(self.id) or duplicates:
+            raise DuplicateTransaction('transaction `{}` already exists'
+                                       .format(self.id))
+
+        if not self.inputs_valid(input_conditions):
+            raise InvalidSignature('Transaction signature is invalid.')
+
+        current_validators = self.get_validators(bigchain)
+
+        # NOTE: Proposer should be a single node
+        if len(self.inputs) != 1 or len(self.inputs[0].owners_before) != 1:
+            raise MultipleInputsError('`tx_signers` must be a list instance of length one')
+
+        # NOTE: Check if the proposer is a validator.
+        [election_initiator_node_pub_key] = self.inputs[0].owners_before
+        if election_initiator_node_pub_key not in current_validators.keys():
+            raise InvalidProposer('Public key is not a part of the validator set')
+
+        # NOTE: Check if all validators have been assigned votes equal to their voting power
+        if not self.is_same_topology(current_validators, self.outputs):
+            raise UnequalValidatorSet('Validator set much be exactly same to the outputs of election')
+
+        return self
+
+    @classmethod
+    def generate(cls, initiator, voters, election_data, metadata=None):
+        # Break symmetry in case we need to call an election with the same properties twice
+        uuid = uuid4()
+        election_data['seed'] = str(uuid)
+
+        (inputs, outputs) = cls.validate_create(initiator, voters, election_data, metadata)
+        election = cls(cls.OPERATION, {'data': election_data}, inputs, outputs, metadata)
+        cls.validate_schema(election.to_dict())
+        return election
+
+    @classmethod
+    def validate_schema(cls, tx):
+        """Validate the election transaction. Since `ELECTION` extends `CREATE` transaction, all the validations for
+        `CREATE` transaction should be inherited
+        """
+        _validate_schema(TX_SCHEMA_COMMON, tx)
+        _validate_schema(TX_SCHEMA_CREATE, tx)
+        if cls.TX_SCHEMA_CUSTOM:
+            _validate_schema(cls.TX_SCHEMA_CUSTOM, tx)
+
+    @classmethod
+    def create(cls, tx_signers, recipients, metadata=None, asset=None):
+        raise NotImplementedError
+
+    @classmethod
+    def transfer(cls, tx_signers, recipients, metadata=None, asset=None):
+        raise NotImplementedError
+
+    @classmethod
+    def to_public_key(cls, election_id):
+        return base58.b58encode(bytes.fromhex(election_id)).decode()
+
+    @classmethod
+    def count_votes(cls, election_pk, transactions, getter=getattr):
+        votes = 0
+        for txn in transactions:
+            if getter(txn, 'operation') == Vote.OPERATION:
+                for output in getter(txn, 'outputs'):
+                    # NOTE: We enforce that a valid vote to election id will have only
+                    # election_pk in the output public keys, including any other public key
+                    # along with election_pk will lead to vote being not considered valid.
+                    if len(getter(output, 'public_keys')) == 1 and [election_pk] == getter(output, 'public_keys'):
+                        votes = votes + int(getter(output, 'amount'))
+        return votes
+
+    def get_commited_votes(self, bigchain, election_pk=None):
+        if election_pk is None:
+            election_pk = self.to_public_key(self.id)
+        txns = list(backend.query.get_asset_tokens_for_public_key(bigchain.connection,
+                                                                  self.id,
+                                                                  election_pk))
+        return self.count_votes(election_pk, txns, dict.get)
+
+    def has_concluded(self, bigchain, current_votes=[]):
+        """Check if the election can be concluded or not.
+
+        * Elections can only be concluded if the validator set has not changed
+          since the election was initiated.
+        * Elections can be concluded only if the current votes form a supermajority.
+
+        Custom elections may override this function and introduce additional checks.
+        """
+        if self.has_validator_set_changed(bigchain):
+            return False
+
+        election_pk = self.to_public_key(self.id)
+        votes_committed = self.get_commited_votes(bigchain, election_pk)
+        votes_current = self.count_votes(election_pk, current_votes)
+
+        total_votes = sum(output.amount for output in self.outputs)
+        if (votes_committed < (2/3) * total_votes) and \
+                (votes_committed + votes_current >= (2/3)*total_votes):
+            return True
+
+        return False
+
+    def get_status(self, bigchain):
+        election = self.get_election(self.id, bigchain)
+        if election and election['is_concluded']:
+            return self.CONCLUDED
+
+        return self.INCONCLUSIVE if self.has_validator_set_changed(bigchain) else self.ONGOING
+
+    def has_validator_set_changed(self, bigchain):
+        latest_change = self.get_validator_change(bigchain)
+        if latest_change is None:
+            return False
+
+        latest_change_height = latest_change['height']
+
+        election = self.get_election(self.id, bigchain)
+
+        return latest_change_height > election['height']
+
+    def get_election(self, election_id, bigchain):
+        return bigchain.get_election(election_id)
+
+    def store(self, bigchain, height, is_concluded):
+        bigchain.store_election(self.id, height, is_concluded)
+
+    def show_election(self, bigchain):
+        data = self.asset['data']
+        if 'public_key' in data.keys():
+            data['public_key'] = public_key_to_base64(data['public_key']['value'])
+        response = ''
+        for k, v in data.items():
+            if k != 'seed':
+                response += f'{k}={v}\n'
+        response += f'status={self.get_status(bigchain)}'
+
+        return response
+
+    @classmethod
+    def _get_initiated_elections(cls, height, txns):
+        elections = []
+        for tx in txns:
+            if not isinstance(tx, Election):
+                continue
+
+            elections.append({'election_id': tx.id, 'height': height,
+                              'is_concluded': False})
+        return elections
+
+    @classmethod
+    def _get_votes(cls, txns):
+        elections = OrderedDict()
+        for tx in txns:
+            if not isinstance(tx, Vote):
+                continue
+
+            election_id = tx.asset['id']
+            if election_id not in elections:
+                elections[election_id] = []
+            elections[election_id].append(tx)
+        return elections
+
+    @classmethod
+    def process_block(cls, bigchain, new_height, txns):
+        """Looks for election and vote transactions inside the block, records
+           and processes elections.
+
+           Every election is recorded in the database.
+
+           Every vote has a chance to conclude the corresponding election. When
+           an election is concluded, the corresponding database record is
+           marked as such.
+
+           Elections and votes are processed in the order in which they
+           appear in the block. Elections are concluded in the order of
+           appearance of their first votes in the block.
+
+           For every election concluded in the block, calls its `on_approval`
+           method. The returned value of the last `on_approval`, if any,
+           is a validator set update to be applied in one of the following blocks.
+
+           `on_approval` methods are implemented by elections of particular type.
+           The method may contain side effects but should be idempotent. To account
+           for other concluded elections, if it requires so, the method should
+           rely on the database state.
+        """
+        # elections initiated in this block
+        initiated_elections = cls._get_initiated_elections(new_height, txns)
+
+        if initiated_elections:
+            bigchain.store_elections(initiated_elections)
+
+        # elections voted for in this block and their votes
+        elections = cls._get_votes(txns)
+
+        validator_update = None
+        for election_id, votes in elections.items():
+            election = bigchain.get_transaction(election_id)
+            if election is None:
+                continue
+
+            if not election.has_concluded(bigchain, votes):
+                continue
+
+            validator_update = election.on_approval(bigchain, new_height)
+            election.store(bigchain, new_height, is_concluded=True)
+
+        return [validator_update] if validator_update else []
+
+    @classmethod
+    def rollback(cls, bigchain, new_height, txn_ids):
+        """Looks for election and vote transactions inside the block and
+           cleans up the database artifacts possibly created in `process_blocks`.
+
+           Part of the `end_block`/`commit` crash recovery.
+        """
+
+        # delete election records for elections initiated at this height and
+        # elections concluded at this height
+        bigchain.delete_elections(new_height)
+
+        txns = [bigchain.get_transaction(tx_id) for tx_id in txn_ids]
+
+        elections = cls._get_votes(txns)
+        for election_id in elections:
+            election = bigchain.get_transaction(election_id)
+            election.on_rollback(bigchain, new_height)
+
+    def on_approval(self, bigchain, new_height):
+        """Override to update the database state according to the
+           election rules. Consider the current database state to account for
+           other concluded elections, if required.
+        """
+        raise NotImplementedError
+
+    def on_rollback(self, bigchain, new_height):
+        """Override to clean up the database artifacts possibly created
+           in `on_approval`. Part of the `end_block`/`commit` crash recovery.
+        """
+        raise NotImplementedError

bigchaindb/elections/vote.py

@@ -0,0 +1,64 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+from bigchaindb.common.transaction import Transaction
+from bigchaindb.common.schema import (_validate_schema,
+                                      TX_SCHEMA_COMMON,
+                                      TX_SCHEMA_TRANSFER,
+                                      TX_SCHEMA_VOTE)
+
+
+class Vote(Transaction):
+
+    OPERATION = 'VOTE'
+    # NOTE: This class inherits TRANSFER txn type. The `TRANSFER` property is
+    # overriden to re-use methods from parent class
+    TRANSFER = OPERATION
+    ALLOWED_OPERATIONS = (OPERATION,)
+    # Custom validation schema
+    TX_SCHEMA_CUSTOM = TX_SCHEMA_VOTE
+
+    def validate(self, bigchain, current_transactions=[]):
+        """Validate election vote transaction
+        NOTE: There are no additional validity conditions on casting votes i.e.
+        a vote is just a valid TRANFER transaction
+
+        For more details refer BEP-21: https://github.com/bigchaindb/BEPs/tree/master/21
+
+        Args:
+            bigchain (BigchainDB): an instantiated bigchaindb.lib.BigchainDB object.
+
+        Returns:
+            Vote: a Vote object
+
+        Raises:
+            ValidationError: If the election vote is invalid
+        """
+        self.validate_transfer_inputs(bigchain, current_transactions)
+        return self
+
+    @classmethod
+    def generate(cls, inputs, recipients, election_id, metadata=None):
+        (inputs, outputs) = cls.validate_transfer(inputs, recipients, election_id, metadata)
+        election_vote = cls(cls.OPERATION, {'id': election_id}, inputs, outputs, metadata)
+        cls.validate_schema(election_vote.to_dict())
+        return election_vote
+
+    @classmethod
+    def validate_schema(cls, tx):
+        """Validate the validator election vote transaction. Since `VOTE` extends `TRANSFER`
+           transaction, all the validations for `CREATE` transaction should be inherited
+        """
+        _validate_schema(TX_SCHEMA_COMMON, tx)
+        _validate_schema(TX_SCHEMA_TRANSFER, tx)
+        _validate_schema(cls.TX_SCHEMA_CUSTOM, tx)
+
+    @classmethod
+    def create(cls, tx_signers, recipients, metadata=None, asset=None):
+        raise NotImplementedError
+
+    @classmethod
+    def transfer(cls, tx_signers, recipients, metadata=None, asset=None):
+        raise NotImplementedError

bigchaindb/events.py

@@ -1,33 +1,109 @@
-from enum import Enum
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+from queue import Empty
+from collections import defaultdict
 from multiprocessing import Queue
 
 
-class EventTypes(Enum):
+POISON_PILL = 'POISON_PILL'
+
+
+class EventTypes:
+    """Container class that holds all the possible
+    events BigchainDB manages.
+    """
+
+    # If you add a new Event Type, make sure to add it
+    # to the docs in docs/server/source/event-plugin-api.rst
+    ALL = ~0
     BLOCK_VALID = 1
     BLOCK_INVALID = 2
+    # NEW_EVENT = 4
+    # NEW_EVENT = 8
+    # NEW_EVENT = 16...
 
 
 class Event:
+    """An Event."""
 
     def __init__(self, event_type, event_data):
+        """Creates a new event.
+
+        Args:
+            event_type (int): the type of the event, see
+                :class:`~bigchaindb.events.EventTypes`
+            event_data (obj): the data of the event.
+        """
+
         self.type = event_type
         self.data = event_data
 
 
-class EventHandler:
+class Exchange:
+    """Dispatch events to subscribers."""
 
-    def __init__(self, events_queue):
-        self.events_queue = events_queue
+    def __init__(self):
+        self.publisher_queue = Queue()
+        self.started_queue = Queue()
 
-    def put_event(self, event, timeout=None):
-        # TODO: handle timeouts
-        self.events_queue.put(event, timeout=None)
+        # Map <event_types -> queues>
+        self.queues = defaultdict(list)
 
-    def get_event(self, timeout=None):
-        # TODO: handle timeouts
-        return self.events_queue.get(timeout=None)
+    def get_publisher_queue(self):
+        """Get the queue used by the publisher.
 
+        Returns:
+            a :class:`multiprocessing.Queue`.
+        """
 
-def setup_events_queue():
-    # TODO: set bounds to the queue
-    return Queue()
+        return self.publisher_queue
+
+    def get_subscriber_queue(self, event_types=None):
+        """Create a new queue for a specific combination of event types
+        and return it.
+
+        Returns:
+            a :class:`multiprocessing.Queue`.
+        Raises:
+            RuntimeError if called after `run`
+        """
+
+        try:
+            self.started_queue.get(timeout=1)
+            raise RuntimeError('Cannot create a new subscriber queue while Exchange is running.')
+        except Empty:
+            pass
+
+        if event_types is None:
+            event_types = EventTypes.ALL
+
+        queue = Queue()
+        self.queues[event_types].append(queue)
+        return queue
+
+    def dispatch(self, event):
+        """Given an event, send it to all the subscribers.
+
+        Args
+            event (:class:`~bigchaindb.events.EventTypes`): the event to
+                dispatch to all the subscribers.
+        """
+
+        for event_types, queues in self.queues.items():
+            if event.type & event_types:
+                for queue in queues:
+                    queue.put(event)
+
+    def run(self):
+        """Start the exchange"""
+        self.started_queue.put('STARTED')
+
+        while True:
+            event = self.publisher_queue.get()
+            if event == POISON_PILL:
+                return
+            else:
+                self.dispatch(event)

bigchaindb/exceptions.py

@@ -1,14 +1,12 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+
 class BigchainDBError(Exception):
     """Base class for BigchainDB exceptions."""
 
 
 class CriticalDoubleSpend(BigchainDBError):
     """Data integrity error that requires attention"""
-
-
-class CriticalDoubleInclusion(BigchainDBError):
-    """Data integrity error that requires attention"""
-
-
-class CriticalDuplicateVote(BigchainDBError):
-    """Data integrity error that requires attention"""

bigchaindb/fastquery.py

@@ -1,54 +1,22 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# BigchainDB and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
 from bigchaindb.utils import condition_details_has_owner
 from bigchaindb.backend import query
 from bigchaindb.common.transaction import TransactionLink
 
 
-class FastQuery:
-    """
-    Database queries that join on block results from a single node.
+class FastQuery():
+    """Database queries that join on block results from a single node."""
 
-    * Votes are not validated for security (security is a replication concern)
-    * Votes come from only one node, and as such, non-byzantine fault tolerance
-      is reduced.
-
-    Previously, to consider the status of a block, all votes for that block
-    were retrieved and the election results were counted. This meant that a
-    faulty node may still have been able to obtain a correct election result.
-    However, from the point of view of a client, it is still neccesary to
-    query multiple nodes to insure against getting an incorrect response from
-    a byzantine node.
-    """
-    def __init__(self, connection, me):
+    def __init__(self, connection):
         self.connection = connection
-        self.me = me
-
-    def filter_valid_block_ids(self, block_ids, include_undecided=False):
-        """
-        Given block ids, return only the ones that are valid.
-        """
-        block_ids = list(set(block_ids))
-        votes = query.get_votes_for_blocks_by_voter(
-                    self.connection, block_ids, self.me)
-        votes = {vote['vote']['voting_for_block']: vote['vote']['is_block_valid']
-                 for vote in votes}
-        return [block_id for block_id in block_ids
-                if votes.get(block_id, include_undecided)]
-
-    def filter_valid_items(self, items, block_id_key=lambda b: b[0]):
-        """
-        Given items with block ids, return only the ones that are valid or undecided.
-        """
-        items = list(items)
-        block_ids = map(block_id_key, items)
-        valid_block_ids = set(self.filter_valid_block_ids(block_ids, True))
-        return [b for b in items if block_id_key(b) in valid_block_ids]
 
     def get_outputs_by_public_key(self, public_key):
-        """
-        Get outputs for a public key
-        """
-        res = list(query.get_owned_ids(self.connection, public_key))
-        txs = [tx for _, tx in self.filter_valid_items(res)]
+        """Get outputs for a public key"""
+        txs = list(query.get_owned_ids(self.connection, public_key))
         return [TransactionLink(tx['id'], index)
                 for tx in txs
                 for index, output in enumerate(tx['outputs'])
@@ -56,30 +24,26 @@ class FastQuery:
                                                public_key)]
 
     def filter_spent_outputs(self, outputs):
-        """
-        Remove outputs that have been spent
+        """Remove outputs that have been spent
 
         Args:
             outputs: list of TransactionLink
         """
         links = [o.to_dict() for o in outputs]
-        res = query.get_spending_transactions(self.connection, links)
-        txs = [tx for _, tx in self.filter_valid_items(res)]
+        txs = list(query.get_spending_transactions(self.connection, links))
         spends = {TransactionLink.from_dict(input_['fulfills'])
                   for tx in txs
                   for input_ in tx['inputs']}
         return [ff for ff in outputs if ff not in spends]
 
     def filter_unspent_outputs(self, outputs):
-        """
-        Remove outputs that have not been spent
+        """Remove outputs that have not been spent
 
         Args:
             outputs: list of TransactionLink
         """
         links = [o.to_dict() for o in outputs]
-        res = query.get_spending_transactions(self.connection, links)
-        txs = [tx for _, tx in self.filter_valid_items(res)]
+        txs = list(query.get_spending_transactions(self.connection, links))
         spends = {TransactionLink.from_dict(input_['fulfills'])
                   for tx in txs
                   for input_ in tx['inputs']}