diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5dc7002a..92137572 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,8 +39,8 @@ Familiarize yourself with how we do coding and documentation in the BigchainDB p * [Install RethinkDB Server](https://rethinkdb.com/docs/install/) * Make sure you have Python 3.4+ (preferably in a virtualenv) -* [Install BigchaindB Server's OS-level dependencies](http://bigchaindb.readthedocs.io/en/latest/appendices/install-os-level-deps.html) -* [Make sure you have the latest Python 3 version of pip and setuptools](http://bigchaindb.readthedocs.io/en/latest/appendices/install-latest-pip.html) +* [Install BigchaindB Server's OS-level dependencies](https://docs.bigchaindb.com/projects/server/en/latest/appendices/install-os-level-deps.html) +* [Make sure you have the latest Python 3 version of pip and setuptools](https://docs.bigchaindb.com/projects/server/en/latest/appendices/install-latest-pip.html) ### Step 3 - Fork bigchaindb on GitHub diff --git a/Dockerfile-dev b/Dockerfile-dev index a955dd5d..2d0cf7ba 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -10,3 +10,4 @@ RUN pip install --upgrade pip COPY . /usr/src/app/ RUN pip install --no-cache-dir -e .[dev] +RUN bigchaindb -y configure diff --git a/README.md b/README.md index a766efa7..d0d289ce 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI](https://img.shields.io/pypi/v/bigchaindb.svg)](https://pypi.python.org/pypi/BigchainDB) [![Travis branch](https://img.shields.io/travis/bigchaindb/bigchaindb/master.svg)](https://travis-ci.org/bigchaindb/bigchaindb) [![Codecov branch](https://img.shields.io/codecov/c/github/bigchaindb/bigchaindb/master.svg)](https://codecov.io/github/bigchaindb/bigchaindb?branch=master) -[![Documentation Status](https://readthedocs.org/projects/bigchaindb/badge/?version=latest)](https://bigchaindb.readthedocs.org/en/latest/) +[![Documentation Status](https://docs.bigchaindb.com/projects/server/en/latest/?badge=latest)](https://docs.bigchaindb.com/projects/server/en/latest/) [![Join the chat at https://gitter.im/bigchaindb/bigchaindb](https://badges.gitter.im/bigchaindb/bigchaindb.svg)](https://gitter.im/bigchaindb/bigchaindb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -15,11 +15,12 @@ We're hiring! [Learn more](https://github.com/bigchaindb/org/blob/master/engjob. ## Get Started with BigchainDB Server -### [Quickstart](http://bigchaindb.readthedocs.io/en/latest/quickstart.html) -### [Set Up & Run a Dev/Test Node](http://bigchaindb.readthedocs.io/en/latest/dev-and-test/setup-run-node.html) -### [Run BigchainDB Server with Docker](http://bigchaindb.readthedocs.io/en/latest/appendices/run-with-docker.html) +### [Quickstart](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html) +### [Set Up & Run a Dev/Test Node](https://docs.bigchaindb.com/projects/server/en/latest/dev-and-test/setup-run-node.html) +### [Run BigchainDB Server with Docker](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-docker.html) ## Links for Everyone + * [BigchainDB.com](https://www.bigchaindb.com/) - the main BigchainDB website, including newsletter signup * [Whitepaper](https://www.bigchaindb.com/whitepaper/) - outlines the motivations, goals and core algorithms of BigchainDB * [Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) @@ -28,7 +29,9 @@ We're hiring! [Learn more](https://github.com/bigchaindb/org/blob/master/engjob. * [Google Group](https://groups.google.com/forum/#!forum/bigchaindb) ## Links for Developers -* [BigchainDB Server Documentation](http://bigchaindb.readthedocs.io/en/latest/) - for developers + +* [All BigchainDB Documentation](https://docs.bigchaindb.com/en/latest/) +* [BigchainDB Server Documentation](https://docs.bigchaindb.com/projects/server/en/latest/index.html) * [CONTRIBUTING.md](CONTRIBUTING.md) - how to contribute * [Community guidelines](CODE_OF_CONDUCT.md) * [Open issues](https://github.com/bigchaindb/bigchaindb/issues) @@ -36,6 +39,7 @@ We're hiring! [Learn more](https://github.com/bigchaindb/org/blob/master/engjob. * [Gitter chatroom](https://gitter.im/bigchaindb/bigchaindb) ## Legal + * [Licenses](LICENSES.md) - open source & open content * [Imprint](https://www.bigchaindb.com/imprint/) * [Contact Us](https://www.bigchaindb.com/contact/) diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md index 12d08ce2..aadccfdc 100644 --- a/benchmarking-tests/test1/README.md +++ b/benchmarking-tests/test1/README.md @@ -2,7 +2,7 @@ Measure how many blocks per second are created on the _bigchain_ with a pre filled backlog. -1. Deploy an aws cluster http://bigchaindb.readthedocs.io/en/latest/deploy-on-aws.html +1. Deploy an aws cluster https://docs.bigchaindb.com/projects/server/en/latest/clusters-feds/aws-testing-cluster.html 2. Make a symbolic link to hostlist.py: `ln -s ../deploy-cluster-aws/hostlist.py .` 3. Make a symbolic link to bigchaindb.pem: ```bash diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index a89692aa..1276f9a5 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -10,7 +10,7 @@ config = { 'server': { # Note: this section supports all the Gunicorn settings: # - http://docs.gunicorn.org/en/stable/settings.html - 'bind': 'localhost:9984', + 'bind': os.environ.get('BIGCHAINDB_SERVER_BIND') or 'localhost:9984', 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, @@ -29,7 +29,7 @@ config = { 'port': 8125, 'rate': 0.01, }, - 'api_endpoint': 'http://localhost:9984/api/v1', + 'api_endpoint': os.environ.get('BIGCHAINDB_API_ENDPOINT') or 'http://localhost:9984/api/v1', 'backlog_reassign_delay': 30 } diff --git a/bigchaindb/core.py b/bigchaindb/core.py index db9b4947..db0a7555 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -189,7 +189,8 @@ class Bigchain(object): return self.validate_transaction(transaction) except (ValueError, exceptions.OperationError, exceptions.TransactionDoesNotExist, exceptions.TransactionOwnerError, exceptions.DoubleSpend, - exceptions.InvalidHash, exceptions.InvalidSignature): + exceptions.InvalidHash, exceptions.InvalidSignature, + exceptions.FulfillmentNotInValidBlock): return False def get_transaction(self, txid, include_status=False): diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 3b708b78..342d6443 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -2,6 +2,7 @@ from bigchaindb_common.crypto import hash_data, VerifyingKey, SigningKey from bigchaindb_common.exceptions import (InvalidHash, InvalidSignature, OperationError, DoubleSpend, TransactionDoesNotExist, + FulfillmentNotInValidBlock, AssetIdMismatch) from bigchaindb_common.transaction import Transaction, Asset from bigchaindb_common.util import gen_timestamp, serialize @@ -81,11 +82,18 @@ class Transaction(Transaction): for ffill in self.fulfillments: input_txid = ffill.tx_input.txid input_cid = ffill.tx_input.cid - input_tx = bigchain.get_transaction(input_txid) + input_tx, status = bigchain.\ + get_transaction(input_txid, include_status=True) + if input_tx is None: raise TransactionDoesNotExist("input `{}` doesn't exist" .format(input_txid)) + if status != bigchain.TX_VALID: + raise FulfillmentNotInValidBlock( + 'input `{}` does not exist in a valid block'.format( + input_txid)) + spent = bigchain.get_spent(input_txid, ffill.tx_input.cid) if spent and spent.id != self.id: raise DoubleSpend('input `{}` was already spent' @@ -97,7 +105,7 @@ class Transaction(Transaction): # validate asset id asset_id = Asset.get_asset_id(input_txs) if asset_id != self.asset.data_id: - raise AssetIdMismatch('The asset id of the input does not match the asset id of the transaction') + raise AssetIdMismatch('The asset id of the input does not match the asset id of the transaction') else: allowed_operations = ', '.join(Transaction.ALLOWED_OPERATIONS) raise TypeError('`operation`: `{}` must be either {}.' diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index fab7b7db..1ff6d30c 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -1,7 +1,7 @@ """This module provides the blueprint for some basic API endpoints. For more information please refer to the documentation on ReadTheDocs: - - https://bigchaindb.readthedocs.io/en/latest/drivers-clients/http-client-server-api.html + - https://docs.bigchaindb.com/projects/server/en/latest/drivers-clients/http-client-server-api.html """ import flask diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index e1986d0a..6439ff4c 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -1,7 +1,7 @@ """This module provides the blueprint for some basic API endpoints. For more information please refer to the documentation on ReadTheDocs: - - https://bigchaindb.readthedocs.io/en/latest/drivers-clients/http-client-server-api.html + - https://docs.bigchaindb.com/projects/server/en/latest/drivers-clients/http-client-server-api.html """ from flask import current_app, request, Blueprint from flask_restful import Resource, Api diff --git a/deploy-cluster-aws/example_deploy_conf.py b/deploy-cluster-aws/example_deploy_conf.py index d569bf1a..5de1b0f5 100644 --- a/deploy-cluster-aws/example_deploy_conf.py +++ b/deploy-cluster-aws/example_deploy_conf.py @@ -40,20 +40,19 @@ USE_KEYPAIRS_FILE=False # IMAGE_ID is the Amazon Machine Image (AMI) id to use # in all the servers/instances to be launched. -# Examples: -# "ami-accff2b1" = An Ubuntu 14.04.2 LTX "Ubuntu Cloud image" from Canonical -# 64-bit, hvm-ssd, published to eu-central-1 -# See http://tinyurl.com/hkjhg46 -# "ami-596b7235" = Ubuntu with IOPS storage? Does this work? -# -# See http://cloud-images.ubuntu.com/releases/14.04/release-20150325/ -IMAGE_ID="ami-accff2b1" +# Canonical (the company behind Ubuntu) generates many AMIs +# and you can search for one that meets your needs at: +# https://cloud-images.ubuntu.com/locator/ec2/ +# Example: +# "ami-72c33e1d" +# (eu-central-1 Ubuntu 14.04 LTS amd64 hvm:ebs-ssd 20160919) +IMAGE_ID="ami-72c33e1d" # INSTANCE_TYPE is the type of AWS instance to launch # i.e. How many CPUs do you want? How much storage? etc. -# Examples: "m3.2xlarge", "c3.8xlarge", "c4.8xlarge" +# Examples: "t2.medium", "m3.2xlarge", "c3.8xlarge", "c4.8xlarge" # For all options, see https://aws.amazon.com/ec2/instance-types/ -INSTANCE_TYPE="m3.2xlarge" +INSTANCE_TYPE="t2.medium" # SECURITY_GROUP is the name of the AWS security group to use. # That security group must exist. @@ -61,7 +60,7 @@ INSTANCE_TYPE="m3.2xlarge" SECURITY_GROUP="bigchaindb" # USING_EBS is True if you want to attach an Amazon EBS volume -USING_EBS=False +USING_EBS=True # EBS_VOLUME_SIZE is the size of the EBS volume to attach, in GiB # Since we assume 'gp2' volumes (for now), the possible range is 1 to 16384 diff --git a/docker-compose.yml b/docker-compose.yml index a71ef0f0..c30b69b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,10 @@ services: - ./setup.py:/usr/src/app/setup.py - ./setup.cfg:/usr/src/app/setup.cfg - ./pytest.ini:/usr/src/app/pytest.ini - - ~/.bigchaindb_docker:/root/.bigchaindb_docker environment: BIGCHAINDB_DATABASE_HOST: rdb - BIGCHAINDB_CONFIG_PATH: /root/.bigchaindb_docker/config + BIGCHAINDB_API_ENDPOINT: http://bdb:9984/api/v1 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + ports: + - "9984" command: bigchaindb start diff --git a/docs/source/appendices/install-with-lxd.md b/docs/source/appendices/install-with-lxd.md index 7cf5cf81..cb12b4ff 100644 --- a/docs/source/appendices/install-with-lxd.md +++ b/docs/source/appendices/install-with-lxd.md @@ -39,5 +39,3 @@ Now, from the host (and the correct directory) where you saved `install.sh`, run `cat install.sh | lxc exec bigchaindb /bin/bash` If you followed the commands correctly, you will have successfully created an LXC container (using LXD) that can get you up and running with BigchainDB in <5 minutes (depending on how long it takes to download all the packages). - -From this point onwards, you can follow the [Python Example](https://bigchaindb.readthedocs.io/en/latest/drivers-clients/python-server-api-examples.html) . diff --git a/docs/source/appendices/run-with-docker.md b/docs/source/appendices/run-with-docker.md index d763e98a..4747de56 100644 --- a/docs/source/appendices/run-with-docker.md +++ b/docs/source/appendices/run-with-docker.md @@ -38,7 +38,7 @@ Let's analyze that command: documentation](https://docs.docker.com/engine/userguide/containers/dockervolumes/#mount-a-host-directory-as-a-data-volume) * `-t` allocate a pseudo-TTY * `-i` keep STDIN open even if not attached -* `bigchaindb/bigchaindb the image to use +* `bigchaindb/bigchaindb` the image to use * `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values @@ -89,7 +89,8 @@ You should see a container named `bigchaindb` in the list. You can load test the BigchainDB running in that container by running the `bigchaindb load` command in a second container: ```text -docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ +docker run --rm -v "$HOME/bigchaindb_docker:/data" \ + -e BIGCHAINDB_DATABASE_HOST=bigchaindb \ --link bigchaindb \ bigchaindb/bigchaindb load ``` diff --git a/docs/source/clusters-feds/aws-testing-cluster.md b/docs/source/clusters-feds/aws-testing-cluster.md index 6fd85641..16fe27d2 100644 --- a/docs/source/clusters-feds/aws-testing-cluster.md +++ b/docs/source/clusters-feds/aws-testing-cluster.md @@ -126,10 +126,10 @@ BRANCH="master" WHAT_TO_DEPLOY="servers" SSH_KEY_NAME="not-set-yet" USE_KEYPAIRS_FILE=False -IMAGE_ID="ami-accff2b1" -INSTANCE_TYPE="m3.2xlarge" +IMAGE_ID="ami-72c33e1d" +INSTANCE_TYPE="t2.medium" SECURITY_GROUP="bigchaindb" -USING_EBS=False +USING_EBS=True EBS_VOLUME_SIZE=30 EBS_OPTIMIZED=False ``` diff --git a/docs/source/dev-and-test/setup-run-node.md b/docs/source/dev-and-test/setup-run-node.md index 232c73bb..0b55d543 100644 --- a/docs/source/dev-and-test/setup-run-node.md +++ b/docs/source/dev-and-test/setup-run-node.md @@ -38,3 +38,87 @@ The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/b Ian Worrall of [Encrypted Labs](http://www.encryptedlabs.com/) wrote a document (PDF) explaining how to set up a BigchainDB (Server) dev machine on [Cloud9](https://c9.io/): [Download that document from GitHub](https://github.com/bigchaindb/bigchaindb/raw/master/docs/source/_static/cloud9.pdf) + + +## Option C: Using a Local Dev Machine and Docker + +You need to have recent versions of [docker engine](https://docs.docker.com/engine/installation/#installation) +and [docker-compose](https://docs.docker.com/compose/install/). + +Build the images: + +```bash +docker-compose build +``` + +Start RethinkDB: + +```bash +docker-compose up -d rdb +``` + +The RethinkDB web interface should be accessible at . +Depending on which platform, and/or how you are running docker, you may need +to change `localhost` for the `ip` of the machine that is running docker. As a +dummy example, if the `ip` of that machine was `0.0.0.0`, you would accees the +web interface at: . + +Start a BigchainDB node: + +```bash +docker-compose up -d bdb +``` + +You can monitor the logs: + +```bash +docker-compose logs -f bdb +``` + +If you wish to run the tests: + +```bash +docker-compose run --rm bdb py.test -v -n auto +``` + +A quick check to make sure that the BigchainDB server API is operational: + +```bash +curl $(docker-compose port bdb 9984) +``` + +should give you something like: + +```bash +{ + "api_endpoint": "http://bdb:9984/api/v1", + "keyring": [], + "public_key": "Brx8g4DdtEhccsENzNNV6yvQHR8s9ebhKyXPFkWUXh5e", + "software": "BigchainDB", + "version": "0.6.0" +} +``` +How does the above curl command work? Inside the Docker container, BigchainDB +exposes the HTTP API on port `9984`. First we get the public port where that +port is bound: + +```bash +docker-compose port bdb 9984 +``` + +The port binding will change whenever you stop/restart the `bdb` service. You +should get an output similar to: + +```bash +0.0.0.0:32772 +``` + +but with a port different from `32772`. + + +Knowing the public port we can now perform a simple `GET` operation against the +root: + +```bash +curl 0.0.0.0:32772 +``` diff --git a/docs/source/drivers-clients/example-apps.md b/docs/source/drivers-clients/example-apps.md deleted file mode 100644 index e66cffea..00000000 --- a/docs/source/drivers-clients/example-apps.md +++ /dev/null @@ -1,5 +0,0 @@ -# Example Apps - -There are some example BigchainDB apps (i.e. apps which use BigchainDB) in [the GitHub repository named bigchaindb-examples](https://github.com/bigchaindb/bigchaindb-examples). - -As noted there, the examples are for demonstration purposes only and should not be used as-is for production. diff --git a/docs/source/drivers-clients/example-apps.rst b/docs/source/drivers-clients/example-apps.rst new file mode 100644 index 00000000..0aab953e --- /dev/null +++ b/docs/source/drivers-clients/example-apps.rst @@ -0,0 +1,10 @@ +Example Apps +============ + +.. warning:: + + There are some example BigchainDB apps (i.e. apps which use BigchainDB) in the GitHub repository named `bigchaindb-examples `_. They were created before there was much of an HTTP API, so instead of communicating with a BigchainDB node via the HTTP API, they communicate directly with the node using the BigchainDB Python server API and the RethinkDB Python Driver. That's not how a real production app would work. The HTTP API is getting better, and we recommend using it to communicate with BigchainDB nodes. + + Moreover, because of changes to the BigchainDB Server code, some of the examples in the bigchaindb-examples repo might not work anymore, or they might not work as expected. + + In the future, we hope to create a set of examples using the HTTP API (or wrappers of it, such as the Python Driver API). diff --git a/docs/source/drivers-clients/index.rst b/docs/source/drivers-clients/index.rst index 221816bc..428861c8 100644 --- a/docs/source/drivers-clients/index.rst +++ b/docs/source/drivers-clients/index.rst @@ -6,6 +6,6 @@ Drivers & Clients python-server-api-examples http-client-server-api - python-driver-api-examples + python-driver example-apps \ No newline at end of file diff --git a/docs/source/drivers-clients/python-driver-api-examples.md b/docs/source/drivers-clients/python-driver-api-examples.md deleted file mode 100644 index 20015df9..00000000 --- a/docs/source/drivers-clients/python-driver-api-examples.md +++ /dev/null @@ -1,9 +0,0 @@ -# The Python Driver API by Example - -The Python driver API is used by app developers to develop client apps which can communicate with one or more BigchainDB clusters. Under the hood, the Python driver API communicates with the BigchainDB cluster using the BigchainDB HTTP client-server API. - -We've been moving the Python driver code into [its own repository](https://github.com/bigchaindb/bigchaindb-driver) with its own documentation. You can find that documentation at: - -[https://bigchaindb-driver.readthedocs.io/en/latest/index.html](https://bigchaindb-driver.readthedocs.io/en/latest/index.html) - -That documentation will soon have examples. It's a work-in-progress. diff --git a/docs/source/drivers-clients/python-driver.md b/docs/source/drivers-clients/python-driver.md new file mode 100644 index 00000000..99563924 --- /dev/null +++ b/docs/source/drivers-clients/python-driver.md @@ -0,0 +1,7 @@ +# The Python Driver + +The BigchainDB Python Driver is a Python wrapper around the [HTTP Client-Server API](http-client-server-api.html). A developer can use it to develop a Python app that communicates with one or more BigchainDB clusters. + +The BigchainDB Python Driver documentation is at: + +[http://docs.bigchaindb.com/projects/py-driver/en/latest/index.html](http://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5f95f2dc..4371bc97 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,12 +1,10 @@ BigchainDB Server Documentation =============================== -Table of Contents ------------------ - .. toctree:: :maxdepth: 1 + ← Back to All BigchainDB Docs introduction quickstart cloud-deployment-starter-templates/index diff --git a/docs/source/introduction.md b/docs/source/introduction.md index 455f4ada..2b02d964 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -2,35 +2,9 @@ This is the documentation for BigchainDB Server, the BigchainDB software that one runs on servers (but not on clients). -BigchainDB is a scalable blockchain database. That is, it's a "big data" database with some blockchain characteristics, including [decentralization](topic-guides/decentralized.html), [immutability](topic-guides/immutable.html) and [native support for assets](topic-guides/assets.html). +If you want to use BigchainDB Server, then you should first understand what BigchainDB is, plus some of the specialized BigchaindB terminology. You can read about that in [the overall BigchainDB project documentation](https://docs.bigchaindb.com/en/latest/index.html). -You can read about the motivations, goals and high-level architecture in the [BigchainDB whitepaper](https://www.bigchaindb.com/whitepaper/). - - -## Is BigchainDB Production-Ready? - -No, BigchainDB is not production-ready. You can use it to build a prototype or proof-of-concept (POC); many people are already doing that. - -BigchainDB is currently in version 0.X. ([The Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) has the exact version number.) Once we believe that BigchainDB is production-ready, we'll release version 1.0. - -[The BigchainDB Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) will give you a sense of the things we intend to do with BigchainDB in the near term and the long term. - - -## Some Basic Vocabulary - -There is some specialized vocabulary associated with BigchainDB. To get started, you should at least know what what we mean by a BigchainDB *node*, *cluster* and *federation*. - -A **BigchainDB node** is a machine or set of closely-linked machines running RethinkDB Server, BigchainDB Server, and related software. (A "machine" might be a bare-metal server, a virtual machine or a container.) Each node is controlled by one person or organization. - -A set of BigchainDB nodes can connect to each other to form a **cluster**. Each node in the cluster runs the same software. A cluster contains one logical RethinkDB datastore. A cluster may have additional machines to do things such as cluster monitoring. - -The people and organizations that run the nodes in a cluster belong to a **federation** (i.e. another organization). A federation must have some sort of governance structure to make decisions. If a cluster is run by a single company, then the federation is just that company. - -**What's the Difference Between a Cluster and a Federation?** - -A cluster is just a bunch of connected nodes. A federation is an organization which has a cluster, and where each node in the cluster has a different operator. Confusingly, we sometimes call a federation's cluster its "federation." You can probably tell what we mean from context. - -There are several kinds of nodes: +Note that there are a few kinds of nodes: - A **dev/test node** is a node created by a developer working on BigchainDB Server, e.g. for testing new or changed code. A dev/test node is typically run on the developer's local machine. diff --git a/docs/source/server-reference/bigchaindb-cli.md b/docs/source/server-reference/bigchaindb-cli.md index 3e11446a..869ef804 100644 --- a/docs/source/server-reference/bigchaindb-cli.md +++ b/docs/source/server-reference/bigchaindb-cli.md @@ -1,9 +1,6 @@ -# BigchainDB Command Line Interface (CLI) +# Command Line Interface (CLI) -**Note: At the time of writing, BigchainDB Server and our BigchainDB client are combined, so the BigchainDB CLI includes some server-specific commands and some client-specific commands (e.g. `bigchaindb load`). Soon, BigchainDB Server will be separate from all BigchainDB clients, and they'll all have different CLIs.** - - -The command-line command to interact with BigchainDB is `bigchaindb`. +The command-line command to interact with BigchainDB Server is `bigchaindb`. ## bigchaindb \-\-help @@ -70,6 +67,9 @@ Write transactions to the backlog (for benchmarking tests). You can learn more a $ bigchaindb load -h ``` +Note: This command uses the Python Server API to write transactions to the database. It _doesn't_ use the HTTP API or a driver that wraps the HTTP API. + + ## bigchaindb set-shards Set the number of shards in the underlying datastore. For example, the following command will set the number of shards to four: diff --git a/docs/source/server-reference/configuration.md b/docs/source/server-reference/configuration.md index cce5b6b8..41b0c2ca 100644 --- a/docs/source/server-reference/configuration.md +++ b/docs/source/server-reference/configuration.md @@ -1,8 +1,6 @@ -# BigchainDB Configuration Settings +# Configuration Settings -**Note: At the time of writing, BigchainDB Server code and BigchainDB Python driver code are mixed together, so the following settings are the settings used by BigchainDB Server and also by clients written using the Python driver code. Soon, the code will be separated into server, driver and shared modules, so that BigchainDB Server and BigchainDB clients will have different configuration settings.** - -The value of each configuration setting is determined according to the following rules: +The value of each BigchainDB Server configuration setting is determined according to the following rules: * If it's set by an environment variable, then use that value * Otherwise, if it's set in a local config file, then use that value diff --git a/docs/source/topic-guides/assets.md b/docs/source/topic-guides/assets.md deleted file mode 100644 index 3777a996..00000000 --- a/docs/source/topic-guides/assets.md +++ /dev/null @@ -1,20 +0,0 @@ -# How BigchainDB is Good for Asset Registrations & Transfers - -BigchainDB can store data of any kind (within reason), but it's designed to be particularly good for storing asset registrations and transfers: - -* The fundamental thing that one submits to a BigchainDB federation to be checked and stored (if valid) is a _transaction_, and there are two kinds: creation transactions and transfer transactions. -* A creation transaction can be use to register any kind of indivisible asset, along with arbitrary metadata. -* An asset can have zero, one, or several owners. -* The owners of an asset can specify (crypto-)conditions which must be satisified by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a transfer transaction. -* BigchainDB verifies that the conditions have been satisified as part of checking the validity of transfer transactions. (Moreover, anyone can check that they were satisfied.) -* BigchainDB prevents double-spending of an asset. -* Validated transactions are strongly tamper-resistant; see [the section about immutability / tamper-resistance](immutable.html). - -You can read more about the details of BigchainDB transactions in [the section on Transaction, Block and Vote Models (data models)](models.html). - - -## BigchainDB Integration with Other Blockchains - -BigchainDB works with the [Interledger protocol](https://interledger.org/), enabling the transfer of assets between BigchainDB and other blockchains, ledgers, and payment systems. - -We’re actively exploring ways that BigchainDB can be used with other blockchains and platforms. diff --git a/docs/source/topic-guides/bft.md b/docs/source/topic-guides/bft.md deleted file mode 100644 index 0255c7b5..00000000 --- a/docs/source/topic-guides/bft.md +++ /dev/null @@ -1,19 +0,0 @@ -# BigchainDB and Byzantine Fault Tolerance - -We have Byzantine fault tolerance (BFT) in our roadmap, as a switch that people can turn on. We anticipate that turning it on will cause a severe dropoff in performance (to gain some extra security). See [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). - -Among the big, industry-used distributed databases in production today (e.g. DynamoDB, Bigtable, MongoDB, Cassandra, Elasticsearch), none of them are BFT. Indeed, almost all wide-area distributed systems in production are not BFT, including military, banking, healthcare, and other security-sensitive systems. - -The are many more practical things that nodes can do to increase security (e.g. firewalls, key management, access controls). - -From a [recent essay by Ken Birman](http://sigops.org/sosp/sosp15/history/05-birman.pdf) (of Cornell): - -> Oh, and with respect to the BFT point: Jim [Gray] felt that real systems fail by crashing [54]. Others have since done studies reinforcing this view, or finding that even crash-failure solutions can sometimes defend against application corruption. One interesting study, reported during a SOSP WIPS session by Ben Reed (one of the co-developers of Zookeeper), found that at Yahoo, Zookeeper itself had never experienced Byzantine faults in a one-year period that they studied closely. - -> [54] Jim Gray. Why Do Computers Stop and What Can Be Done About It? SOSP, 1985. - -Ben Reed never published those results, but Birman wrote more about them in the book *Guide to Reliable Distributed Systems: Building High-Assurance Applications*. From page 358 of that book: - -> But the cloud community, led by Ben Reed and Flavio Junqueira at Yahoo, sees things differently (these are the two inventor’s [sic] of Yahoo’s ZooKeeper service). **They have described informal studies of how applications and machines at Yahoo failed, concluding that the frequency of Byzantine failures was extremely small relative to the frequency of crash failures** [emphasis added]. Sometimes they did see data corruption, but then they often saw it occur in a correlated way that impacted many replicas all at once. And very often they saw failures occur in the client layer, then propagate into the service. BFT techniques tend to be used only within a service, not in the client layer that talks to that service, hence offer no protection against malfunctioning clients. **All of this, Reed and Junqueira conclude, lead to the realization that BFT just does not match the real needs of a cloud computing company like Yahoo, even if the data being managed by a service really is of very high importance** [emphasis added]. Unfortunately, they have not published this study; it was reported at an “outrageous opinions” session at the ACM Symposium on Operating Systems Principles, in 2009. - -> The practical use of the Byzantine protocol raises another concern: The timing assumptions built into the model [i.e. synchronous or partially-synchronous nodes] are not realizable in most computing environments… diff --git a/docs/source/topic-guides/decentralized.md b/docs/source/topic-guides/decentralized.md deleted file mode 100644 index c8a7e7a3..00000000 --- a/docs/source/topic-guides/decentralized.md +++ /dev/null @@ -1,19 +0,0 @@ -# How BigchainDB is Decentralized - -Decentralization means that no one owns or controls everything, and there is no single point of failure. - -Ideally, each node in a BigchainDB cluster is owned and controlled by a different person or organization. Even if the cluster lives within one organization, it's still preferable to have each node controlled by a different person or subdivision. - -We use the phrase "BigchainDB federation" (or just "federation") to refer to the set of people and/or organizations who run the nodes of a BigchainDB cluster. A federation requires some form of governance to make decisions such as membership and policies. The exact details of the governance process are determined by each federation, but it can be very decentralized (e.g. purely vote-based, where each node gets a vote, and there are no special leadership roles). - -The actual data is decentralized in that it doesn’t all get stored in one place. Each federation node stores the primary of one shard and replicas of some other shards. (A shard is a subset of the total set of documents.) Sharding and replication are handled by RethinkDB. - -A federation can increase its decentralization (and its resilience) by increasing its jurisdictional diversity, geographic diversity, and other kinds of diversity. This idea is expanded upon in [the section on node diversity](diversity.html). - -There’s no node that has a long-term special position in the federation. All nodes run the same software and perform the same duties. - -RethinkDB has an “admin” user which can’t be deleted and which can make big changes to the database, such as dropping a table. Right now, that’s a big security vulnerability, but we have plans to mitigate it by: -1. Locking down the admin user as much as possible. -2. Having all nodes inspect RethinkDB admin-type requests before acting on them. Requests can be checked against an evolving whitelist of allowed actions (voted on by federation nodes). - -It’s worth noting that the RethinkDB admin user can’t transfer assets, even today. The only way to create a valid transfer transaction is to fulfill the current (crypto) conditions on the asset, and the admin user can’t do that because the admin user doesn’t have the necessary private keys (or preimages, in the case of hashlock conditions). They’re not stored in the database. diff --git a/docs/source/topic-guides/diversity.md b/docs/source/topic-guides/diversity.md deleted file mode 100644 index 4819a0af..00000000 --- a/docs/source/topic-guides/diversity.md +++ /dev/null @@ -1,11 +0,0 @@ -# Kinds of Node Diversity - -Steps should be taken to make it difficult for any one actor or event to control or damage “enough” of the nodes. (“Enough” is usually a quorum.) There are many kinds of diversity to consider, listed below. It may be quite difficult to have high diversity of all kinds. - -1. **Jurisdictional diversity.** The nodes should be controlled by entities within multiple legal jurisdictions, so that it becomes difficult to use legal means to compel enough of them to do something. -2. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems. -3. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes. -4. **Operating system diversity.** The servers should use a variety of operating systems, so that a security bug in one OS can’t be used to exploit enough of the nodes. -5. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a federation. For example, it provides the federation with a source of various ideas for addressing challenges. - -Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a federation could also have a diversity of server implementations. diff --git a/docs/source/topic-guides/immutable.md b/docs/source/topic-guides/immutable.md deleted file mode 100644 index 388887be..00000000 --- a/docs/source/topic-guides/immutable.md +++ /dev/null @@ -1,17 +0,0 @@ -# How BigchainDB is Immutable / Tamper-Resistant - -The blockchain community often describes blockchains as “immutable.” If we interpret that word literally, it means that blockchain data is unchangeable or permanent, which is absurd. The data _can_ be changed. For example, a plague might drive humanity extinct; the data would then get corrupted over time due to water damage, thermal noise, and the general increase of entropy. In the case of Bitcoin, nothing so drastic is required: a 51% attack will suffice. - -It’s true that blockchain data is more difficult to change than usual: it’s more tamper-resistant than a typical file system or database. Therefore, in the context of blockchains, we interpret the word “immutable” to mean tamper-resistant. (Linguists would say that the word “immutable” is a _term of art_ in the blockchain community.) - -BigchainDB achieves strong tamper-resistance in the following ways: - -1. **Replication.** All data is sharded and shards are replicated in several (different) places. The replication factor can be set by the federation. The higher the replication factor, the more difficult it becomes to change or delete all replicas. -2. **Internal watchdogs.** All nodes monitor all changes and if some unallowed change happens, then appropriate action is taken. For example, if a valid block is deleted, then it is put back. -3. **External watchdogs.** Federations may opt to have trusted third-parties to monitor and audit their data, looking for irregularities. For federations with publicly-readable data, the public can act as an auditor. -4. **Cryptographic signatures** are used throughout BigchainDB as a way to check if messages (transactions, blocks and votes) have been tampered with enroute, and as a way to verify who signed the messages. Each block is signed by the node that created it. Each vote is signed by the node that cast it. A creation transaction is signed by the node that created it, although there are plans to improve that by adding signatures from the sending client and multiple nodes; see [Issue #347](https://github.com/bigchaindb/bigchaindb/issues/347). Transfer transactions can contain multiple fulfillments (one per asset transferred). Each fulfillment will typically contain one or more signatures from the owners (i.e. the owners before the transfer). Hashlock fulfillments are an exception; there’s an open issue ([#339](https://github.com/bigchaindb/bigchaindb/issues/339)) to address that. -5. **Full or partial backups** of the database may be recorded from time to time, possibly on magnetic tape storage, other blockchains, printouts, etc. -6. **Strong security.** Node owners can adopt and enforce strong security policies. -7. **Node diversity.** Diversity makes it so that no one thing (e.g. natural disaster or operating system bug) can compromise enough of the nodes. See [the section on the kinds of node diversity](diversity.html). - -Some of these things come "for free" as part of the BigchainDB software, and others require some extra effort from the federation and node owners. diff --git a/docs/source/topic-guides/index.rst b/docs/source/topic-guides/index.rst index ca308017..9386fe87 100644 --- a/docs/source/topic-guides/index.rst +++ b/docs/source/topic-guides/index.rst @@ -1,19 +1,12 @@ -.. You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - Topic Guides ============ -Topic guides give background and explain concepts at a high level. +.. note:: + + Most of the Topic Guides have been moved over to `the root BigchainDB project docs `_. + .. toctree:: :maxdepth: 1 - decentralized - diversity - immutable - bft - assets - smart-contracts models - timestamps diff --git a/docs/source/topic-guides/models.md b/docs/source/topic-guides/models.md index fd6b20a3..fabf331d 100644 --- a/docs/source/topic-guides/models.md +++ b/docs/source/topic-guides/models.md @@ -84,7 +84,7 @@ Here's some explanation of the contents of a transaction: - `conditions`: List of conditions. Each _condition_ is a _crypto-condition_ that needs to be fulfilled by a transfer transaction in order to transfer ownership to new owners. See [Conditions and Fulfillments](#conditions-and-fulfillments) below. - `operation`: String representation of the operation being performed (currently either "CREATE" or "TRANSFER"). It determines how the transaction should be validated. - - `timestamp`: The Unix time when the transaction was created. It's provided by the client. See [the section on timestamps](timestamps.html). + - `timestamp`: The Unix time when the transaction was created. It's provided by the client. See the page about [timestamps in BigchainDB](https://docs.bigchaindb.com/en/latest/timestamps.html). - `data`: - `uuid`: UUID version 4 (random) converted to a string of hex digits in standard form. - `payload`: Can be any JSON document. It may be empty in the case of a transfer transaction. @@ -245,7 +245,7 @@ If there is only one _current owner_, the fulfillment will be a simple signature - `id`: The hash of the serialized `block` (i.e. the `timestamp`, `transactions`, `node_pubkey`, and `voters`). This is also a database primary key; that's how we ensure that all blocks are unique. - `block`: - - `timestamp`: The Unix time when the block was created. It's provided by the node that created the block. See [the section on timestamps](timestamps.html). + - `timestamp`: The Unix time when the block was created. It's provided by the node that created the block. See the page about [timestamps in BigchainDB](https://docs.bigchaindb.com/en/latest/timestamps.html). - `transactions`: A list of the transactions included in the block. - `node_pubkey`: The public key of the node that create the block. - `voters`: A list of public keys of federation nodes. Since the size of the diff --git a/docs/source/topic-guides/smart-contracts.md b/docs/source/topic-guides/smart-contracts.md deleted file mode 100644 index b42791d9..00000000 --- a/docs/source/topic-guides/smart-contracts.md +++ /dev/null @@ -1,3 +0,0 @@ -# BigchainDB and Smart Contracts - -BigchainDB isn’t intended for running smart contracts. That said, it can do many of the things that smart contracts are used to do. For example, the owners of an asset can impose conditions that must be fulfilled by anyone wishing to transfer the asset to new owners; see the [section on conditions](models.html#conditions-and-fulfillments). BigchainDB also [supports a form of escrow](../drivers-clients/python-server-api-examples.html#escrow). diff --git a/docs/source/topic-guides/timestamps.md b/docs/source/topic-guides/timestamps.md deleted file mode 100644 index 6d76ed77..00000000 --- a/docs/source/topic-guides/timestamps.md +++ /dev/null @@ -1,95 +0,0 @@ -# Timestamps in BigchainDB - -Each transaction, block and vote has an associated timestamp. Interpreting those timestamps is tricky, hence the need for this section. - - -## Timestamp Sources & Accuracy - -A transaction's timestamp is provided by the client which created and submitted the transaction to a BigchainDB node. A block's timestamp is provided by the BigchainDB node which created the block. A vote's timestamp is provided by the BigchainDB node which created the vote. - -When a BigchainDB client or node needs a timestamp, it calls a BigchainDB utility function named `timestamp()`. There's a detailed explanation of how that function works below, but the short version is that it gets the [Unix time](https://en.wikipedia.org/wiki/Unix_time) from its system clock, rounded to the nearest second. - -We can't say anything about the accuracy of the system clock on clients. Timestamps from clients are still potentially useful, however, in a statistical sense. We say more about that below. - -We advise BigchainDB nodes to run special software (an "NTP daemon") to keep their system clock in sync with standard time servers. (NTP stands for [Network Time Protocol](https://en.wikipedia.org/wiki/Network_Time_Protocol).) - - -## Converting Timestamps to UTC - -To convert a BigchainDB timestamp (a Unix time) to UTC, you need to know how the node providing the timestamp was set up. That's because different setups will report a different "Unix time" value around leap seconds! There's [a nice Red Hat Developer Blog post about the various setup options](http://developers.redhat.com/blog/2015/06/01/five-different-ways-handle-leap-seconds-ntp/). If you want more details, see [David Mills' pages about leap seconds, NTP, etc.](https://www.eecis.udel.edu/~mills/leap.html) (David Mills designed NTP.) - -We advise BigchainDB nodes to run an NTP daemon [with particular settings](../appendices/ntp-notes.html), so that their timestamps are consistent. - -If a timestamp comes from a node that's set up as we advise, it can be converted to UTC as follows: - -1. Use a standard "Unix time to UTC" converter to get a UTC timestamp. -2. Is the UTC timestamp a leap second, or the second before/after a leap second? There's [a list of all the leap seconds on Wikipedia](https://en.wikipedia.org/wiki/Leap_second). -3. If no, then you are done. -4. If yes, then it might not be possible to convert it to a single UTC timestamp. Even if it can't be converted to a single UTC timestamp, it _can_ be converted to a list of two possible UTC timestamps. -Showing how to do that is beyond the scope of this documentation. -In all likelihood, you will never have to worry about leap seconds because they are very rare. -(There were only 26 between 1972 and the end of 2015.) - - -## Calculating Elapsed Time Between Two Timestamps - -There's another gotcha with (Unix time) timestamps: you can't calculate the real-world elapsed time between two timestamps (correctly) by subtracting the smaller timestamp from the larger one. The result won't include any of the leap seconds that occured between the two timestamps. You could look up how many leap seconds happened between the two timestamps and add that to the result. There are many library functions for working with timestamps; those are beyond the scope of this documentation. - - -## Avoid Doing Transactions Around Leap Seconds - -Because of the ambiguity and confusion that arises with Unix time around leap seconds, we advise users to avoid creating transactions around leap seconds. - - -## Interpreting Sets of Timestamps - -You can look at many timestamps to get a statistical sense of when something happened. For example, a transaction in a decided-valid block has many associated timestamps: - -* its own timestamp -* the timestamps of the other transactions in the block; there could be as many as 999 of those -* the timestamp of the block -* the timestamps of all the votes on the block - -Those timestamps come from many sources, so you can look at all of them to get some statistical sense of when the transaction "actually happened." The timestamp of the block should always be after the timestamp of the transaction, and the timestamp of the votes should always be after the timestamp of the block. - - -## How BigchainDB Uses Timestamps - -BigchainDB _doesn't_ use timestamps to determine the order of transactions or blocks. In particular, the order of blocks is determined by RethinkDB's changefeed on the bigchain table. - -BigchainDB does use timestamps for some things. It uses them to determine if a transaction has been waiting in the backlog for too long (i.e. because the node assigned to it hasn't handled it yet). It also uses timestamps to determine the status of timeout conditions (used by escrow). - - -## Including Trusted Timestamps - -If you want to create a transaction payload with a trusted timestamp, you can. - -One way to do that would be to send a payload to a trusted timestamping service. They will send back a timestamp, a signature, and their public key. They should also explain how you can verify the signature. You can then include the original payload, the timestamp, the signature, and the service's public key in your transaction. That way, anyone with the verification instructions can verify that the original payload was signed by the trusted timestamping service. - - -## How the timestamp() Function Works - -BigchainDB has a utility function named `timestamp()` which amounts to: -```python -timestamp() = str(round(time.time())) -``` - -In other words, it calls the `time()` function in Python's `time` module, [rounds](https://docs.python.org/3/library/functions.html#round) that to the nearest integer, and converts the result to a string. - -It rounds the output of `time.time()` to the nearest second because, according to [the Python documentation for `time.time()`](https://docs.python.org/3.4/library/time.html#time.time), "...not all systems provide time with a better precision than 1 second." - -How does `time.time()` work? If you look in the C source code, it calls `floattime()` and `floattime()` calls [clock_gettime()](https://www.cs.rutgers.edu/~pxk/416/notes/c-tutorials/gettime.html), if it's available. -```text -ret = clock_gettime(CLOCK_REALTIME, &tp); -``` - -With `CLOCK_REALTIME` as the first argument, it returns the "Unix time." ("Unix time" is in quotes because its value around leap seconds depends on how the system is set up; see above.) - - -## Why Not Use UTC, TAI or Some Other Time that Has Unambiguous Timestamps for Leap Seconds? - -It would be nice to use UTC or TAI timestamps, but unfortunately there's no commonly-available, standard way to get always-accurate UTC or TAI timestamps from the operating system on typical computers today (i.e. accurate around leap seconds). - -There _are_ commonly-available, standard ways to get the "Unix time," such as clock_gettime() function available in C. That's what we use (indirectly via Python). ("Unix time" is in quotes because its value around leap seconds depends on how the system is set up; see above.) - -The Unix-time-based timestamps we use are only ambiguous circa leap seconds, and those are very rare. Even for those timestamps, the extra uncertainty is only one second, and that's not bad considering that we only report timestamps to a precision of one second in the first place. All other timestamps can be converted to UTC with no ambiguity. diff --git a/tests/db/conftest.py b/tests/db/conftest.py index 73760647..3fc6ddce 100644 --- a/tests/db/conftest.py +++ b/tests/db/conftest.py @@ -11,7 +11,9 @@ import rethinkdb as r from bigchaindb import Bigchain from bigchaindb.db import get_conn +from bigchaindb_common import crypto +USER2_SK, USER2_VK = crypto.generate_key_pair() @pytest.fixture(autouse=True) def restore_config(request, node_config): @@ -109,12 +111,13 @@ def inputs(user_vk): # 1. create the genesis block b = Bigchain() try: - b.create_genesis_block() + g = b.create_genesis_block() except GenesisBlockAlreadyExistsError: pass - # 2. create block with transactions for `USER` to spend - for block in range(4): + # 2. create blocks with transactions for `USER` to spend + prev_block_id = g.id + for block in range(4): transactions = [ Transaction.create( [b.me], [user_vk], metadata={'i': i}).sign([b.me_private]) @@ -122,3 +125,46 @@ def inputs(user_vk): ] block = b.create_block(transactions) b.write_block(block, durability='hard') + + # 3. vote the blocks valid, so that the inputs are valid + vote = b.vote(block.id, prev_block_id, True) + prev_block_id = block.id + b.write_vote(vote) + + +@pytest.fixture +def user2_sk(): + return USER2_SK + + +@pytest.fixture +def user2_vk(): + return USER2_VK + + +@pytest.fixture +def inputs_shared(user_vk, user2_vk): + from bigchaindb.models import Transaction + from bigchaindb_common.exceptions import GenesisBlockAlreadyExistsError + # 1. create the genesis block + b = Bigchain() + try: + g = b.create_genesis_block() + except GenesisBlockAlreadyExistsError: + pass + + # 2. create blocks with transactions for `USER` to spend + prev_block_id = g.id + for block in range(4): + transactions = [ + Transaction.create( + [b.me], [user_vk, user2_vk], payload={'i': i}).sign([b.me_private]) + for i in range(10) + ] + block = b.create_block(transactions) + b.write_block(block, durability='hard') + + # 3. vote the blocks valid, so that the inputs are valid + vote = b.vote(block.id, prev_block_id, True) + prev_block_id = block.id + b.write_vote(vote) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 3d420350..c5cf30e2 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1,3 +1,5 @@ +from time import sleep + import pytest @@ -536,7 +538,7 @@ class TestBigchainApi(object): 'input': {'cid': 0, 'txid': 'somethingsomething'}, 'owners_before': [user_vk]} - fulfillment = Fulfillment.from_dict(fulfillment_dict) + fulfillment = Fulfillment.from_dict(fulfillment_dict) tx = Transaction.transfer([fulfillment], [user_vk], Asset()) with pytest.raises(TransactionDoesNotExist) as excinfo: @@ -587,6 +589,7 @@ class TestTransactionValidation(object): with pytest.raises(InvalidSignature): b.validate_transaction(tx) + @pytest.mark.usefixtures('inputs') def test_non_create_double_spend(self, b, signed_create_tx, signed_transfer_tx): from bigchaindb_common.exceptions import DoubleSpend @@ -594,10 +597,20 @@ class TestTransactionValidation(object): block1 = b.create_block([signed_create_tx]) b.write_block(block1) + # vote block valid + vote = b.vote(block1.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + b.write_transaction(signed_transfer_tx) block = b.create_block([signed_transfer_tx]) b.write_block(block, durability='hard') + # vote block valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + + sleep(1) + signed_transfer_tx.timestamp = 123 # FIXME: https://github.com/bigchaindb/bigchaindb/issues/592 with pytest.raises(DoubleSpend): @@ -626,6 +639,33 @@ class TestTransactionValidation(object): # bigchain assert transfer_tx == b.validate_transaction(transfer_tx) + @pytest.mark.usefixtures('inputs') + def test_fulfillment_not_in_valid_block(self, b, user_vk, user_sk): + from bigchaindb.models import Transaction + from bigchaindb_common.exceptions import FulfillmentNotInValidBlock + + input_tx = b.get_owned_ids(user_vk).pop() + input_tx = b.get_transaction(input_tx.txid) + inputs = input_tx.to_inputs() + + # create a transaction that's valid but not in a voted valid block + transfer_tx = Transaction.transfer(inputs, [user_vk], input_tx.asset) + transfer_tx = transfer_tx.sign([user_sk]) + + assert transfer_tx == b.validate_transaction(transfer_tx) + + # create block + block = b.create_block([transfer_tx]) + b.write_block(block, durability='hard') + + # create transaction with the undecided input + tx_invalid = Transaction.transfer(transfer_tx.to_inputs(), [user_vk], + transfer_tx.asset) + tx_invalid = tx_invalid.sign([user_sk]) + + with pytest.raises(FulfillmentNotInValidBlock): + b.validate_transaction(tx_invalid) + class TestBlockValidation(object): @pytest.mark.skipif(reason='Separated tx validation from block creation.') @@ -729,70 +769,55 @@ class TestMultipleInputs(object): @pytest.mark.skipif(reason=('Multiple inputs are only allowed for the ' 'same asset. Remove this after implementing ', 'multiple assets')) + @pytest.mark.usefixtures('inputs') def test_transfer_single_owners_multiple_inputs(self, b, user_sk, user_vk): from bigchaindb_common import crypto from bigchaindb.models import Transaction user2_sk, user2_vk = crypto.generate_key_pair() - # TODO: Make this a fixture - transactions = [] - for i in range(3): - tx = Transaction.create([user_vk], [user_vk]) - tx = tx.sign([user_sk]) - transactions.append(tx) - b.write_transaction(tx) - block = b.create_block(transactions) - b.write_block(block, durability='hard') - # get inputs owned_inputs = b.get_owned_ids(user_vk) input_txs = [b.get_transaction(tx_link.txid) for tx_link in owned_inputs] inputs = sum([input_tx.to_inputs() for input_tx in input_txs], []) - tx = Transaction.transfer(inputs, 3 * [[user_vk]]) + tx = Transaction.transfer(inputs, len(inputs) * [[user_vk]]) tx = tx.sign([user_sk]) assert b.validate_transaction(tx) == tx - assert len(tx.fulfillments) == 3 - assert len(tx.conditions) == 3 + assert len(tx.fulfillments) == len(inputs) + assert len(tx.conditions) == len(inputs) @pytest.mark.skipif(reason=('Multiple inputs are only allowed for the ' 'same asset. Remove this after implementing ', 'multiple assets')) + @pytest.mark.usefixtures('inputs') def test_transfer_single_owners_single_input_from_multiple_outputs(self, b, user_sk, user_vk): - import random from bigchaindb_common import crypto from bigchaindb.models import Transaction user2_sk, user2_vk = crypto.generate_key_pair() - transactions = [] - for i in range(3): - payload = {'somedata': random.randint(0, 255)} - tx = Transaction.create([user_vk], [user_vk], None, 'CREATE', payload) - tx = tx.sign([user_sk]) - transactions.append(tx) - b.write_transaction(tx) - block = b.create_block(transactions) - b.write_block(block, durability='hard') - # get inputs owned_inputs = b.get_owned_ids(user_vk) input_txs = [b.get_transaction(tx_link.txid) for tx_link in owned_inputs] inputs = sum([input_tx.to_inputs() for input_tx in input_txs], []) - tx = Transaction.transfer(inputs, 3 * [[user2_vk]]) + tx = Transaction.transfer(inputs, len(inputs) * [[user2_vk]]) tx = tx.sign([user_sk]) # create block with the transaction block = b.create_block([tx]) b.write_block(block, durability='hard') + # vote block valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + # get inputs from user2 owned_inputs = b.get_owned_ids(user2_vk) - assert len(owned_inputs) == 3 + assert len(owned_inputs) == len(inputs) # create a transaction with a single input from a multiple output transaction tx_link = owned_inputs.pop() @@ -827,40 +852,38 @@ class TestMultipleInputs(object): @pytest.mark.skipif(reason=('Multiple inputs are only allowed for the ' 'same asset. Remove this after implementing ', 'multiple assets')) + @pytest.mark.usefixtures('inputs') def test_single_owner_before_multiple_owners_after_multiple_inputs(self, b, user_sk, user_vk): - import random from bigchaindb_common import crypto from bigchaindb.models import Transaction user2_sk, user2_vk = crypto.generate_key_pair() user3_sk, user3_vk = crypto.generate_key_pair() - transactions = [] - for i in range(3): - payload = {'somedata': random.randint(0, 255)} - tx = Transaction.create([user_vk], [user_vk], None, 'CREATE', - payload) - tx = tx.sign([user_sk]) - transactions.append(tx) - b.write_transaction(tx) - block = b.create_block(transactions) - b.write_block(block, durability='hard') - owned_inputs = b.get_owned_ids(user_vk) input_txs = [b.get_transaction(tx_link.txid) for tx_link in owned_inputs] inputs = sum([input_tx.to_inputs() for input_tx in input_txs], []) - tx = Transaction.transfer(inputs, 3 * [[user2_vk, user3_vk]]) + tx = Transaction.transfer(inputs, len(inputs) * [[user2_vk, user3_vk]]) tx = tx.sign([user_sk]) + # create block with the transaction + block = b.create_block([tx]) + b.write_block(block, durability='hard') + + # vote block valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + # validate transaction assert b.is_valid_transaction(tx) == tx - assert len(tx.fulfillments) == 3 - assert len(tx.conditions) == 3 + assert len(tx.fulfillments) == len(inputs) + assert len(tx.conditions) == len(inputs) + @pytest.mark.usefixtures('inputs') def test_multiple_owners_before_single_owner_after_single_input(self, b, user_sk, user_vk): @@ -875,6 +898,10 @@ class TestMultipleInputs(object): block = b.create_block([tx]) b.write_block(block, durability='hard') + # vote block valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + owned_input = b.get_owned_ids(user_vk).pop() input_tx = b.get_transaction(owned_input.txid) inputs = input_tx.to_inputs() @@ -890,23 +917,15 @@ class TestMultipleInputs(object): @pytest.mark.skipif(reason=('Multiple inputs are only allowed for the ' 'same asset. Remove this after implementing ', 'multiple assets')) + @pytest.mark.usefixtures('inputs_shared') def test_multiple_owners_before_single_owner_after_multiple_inputs(self, b, - user_sk, - user_vk): + user_sk, user_vk, user2_vk, user2_sk): from bigchaindb_common import crypto from bigchaindb.models import Transaction - user2_sk, user2_vk = crypto.generate_key_pair() + # create a new users user3_sk, user3_vk = crypto.generate_key_pair() - transactions = [] - for i in range(3): - tx = Transaction.create([b.me], [user_vk, user2_vk]) - tx = tx.sign([b.me_private]) - transactions.append(tx) - block = b.create_block(transactions) - b.write_block(block, durability='hard') - tx_links = b.get_owned_ids(user_vk) inputs = sum([b.get_transaction(tx_link.txid).to_inputs() for tx_link in tx_links], []) @@ -915,9 +934,10 @@ class TestMultipleInputs(object): tx = tx.sign([user_sk, user2_sk]) assert b.is_valid_transaction(tx) == tx - assert len(tx.fulfillments) == 3 - assert len(tx.conditions) == 3 + assert len(tx.fulfillments) == len(inputs) + assert len(tx.conditions) == len(inputs) + @pytest.mark.usefixtures('inputs') def test_multiple_owners_before_multiple_owners_after_single_input(self, b, user_sk, user_vk): @@ -933,6 +953,11 @@ class TestMultipleInputs(object): block = b.create_block([tx]) b.write_block(block, durability='hard') + # vote block valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + + # get input tx_link = b.get_owned_ids(user_vk).pop() tx_input = b.get_transaction(tx_link.txid) @@ -946,25 +971,17 @@ class TestMultipleInputs(object): @pytest.mark.skipif(reason=('Multiple inputs are only allowed for the ' 'same asset. Remove this after implementing ', 'multiple assets')) - def test_multiple_owners_before_multiple_owners_after_multiple_inputs(self, - b, - user_sk, - user_vk): + @pytest.mark.usefixtures('inputs_shared') + def test_multiple_owners_before_multiple_owners_after_multiple_inputs(self, b, + user_sk, user_vk, + user2_sk, user2_vk): from bigchaindb_common import crypto from bigchaindb.models import Transaction - user2_sk, user2_vk = crypto.generate_key_pair() + # create a new users user3_sk, user3_vk = crypto.generate_key_pair() user4_sk, user4_vk = crypto.generate_key_pair() - transactions = [] - for i in range(3): - tx = Transaction.create([b.me], [user_vk, user2_vk]) - tx = tx.sign([b.me_private]) - transactions.append(tx) - block = b.create_block(transactions) - b.write_block(block, durability='hard') - tx_links = b.get_owned_ids(user_vk) inputs = sum([b.get_transaction(tx_link.txid).to_inputs() for tx_link in tx_links], []) @@ -973,8 +990,8 @@ class TestMultipleInputs(object): tx = tx.sign([user_sk, user2_sk]) assert b.is_valid_transaction(tx) == tx - assert len(tx.fulfillments) == 3 - assert len(tx.conditions) == 3 + assert len(tx.fulfillments) == len(inputs) + assert len(tx.conditions) == len(inputs) def test_get_owned_ids_single_tx_single_output(self, b, user_sk, user_vk): from bigchaindb_common import crypto diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index 65d9d0e5..09811b0d 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -1,3 +1,5 @@ +import time + from unittest.mock import patch import rethinkdb as r @@ -285,9 +287,10 @@ def test_valid_block_voting_with_transfer_transactions(monkeypatch, b): vote_pipeline = vote.create_pipeline() vote_pipeline.setup(indata=inpipe, outdata=outpipe) - inpipe.put(block.to_dict()) - inpipe.put(block2.to_dict()) vote_pipeline.start() + inpipe.put(block.to_dict()) + time.sleep(1) + inpipe.put(block2.to_dict()) vote_out = outpipe.get() vote2_out = outpipe.get() vote_pipeline.terminate() diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 5920ce27..5ffd5b9a 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -121,6 +121,8 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname', 'BIGCHAINDB_DATABASE_PORT': '4242', + 'BIGCHAINDB_API_ENDPOINT': 'api://ipa', + 'BIGCHAINDB_SERVER_BIND': '1.2.3.4:56', 'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'}) import bigchaindb @@ -130,7 +132,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): assert bigchaindb.config == { 'CONFIGURED': True, 'server': { - 'bind': 'localhost:9984', + 'bind': '1.2.3.4:56', 'workers': None, 'threads': None, }, @@ -149,7 +151,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): 'port': 8125, 'rate': 0.01, }, - 'api_endpoint': 'http://localhost:9984/api/v1', + 'api_endpoint': 'api://ipa', 'backlog_reassign_delay': 5 }