diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh index cc1061fe..97cfc977 100755 --- a/.ci/travis_script.sh +++ b/.ci/travis_script.sh @@ -7,11 +7,14 @@ if [[ -n ${TOXENV} ]]; then elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then # Run the full suite of tests for MongoDB over an unsecure connection - pytest -sv --database-backend=mongodb --cov=bigchaindb + pytest -sv --database-backend=mongodb -m "serial" + pytest -sv --database-backend=mongodb --cov=bigchaindb -m "not serial" elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'. pytest -sv --database-backend=mongodb-ssl --cov=bigchaindb -m bdb_ssl else - pytest -sv -n auto --cov=bigchaindb + # Run the full suite of tests for RethinkDB (the default backend when testing) + pytest -sv -m "serial" + pytest -sv --cov=bigchaindb -m "not serial" fi diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..31636a13 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,34 @@ +## Description +A few sentences describing the overall goals of the pull request's commits. + +## Issues This PR Fixes +Fixes #NNNN +Fixes #NNNN + +## Related PRs +List related PRs against other branches e.g. for backporting features/bugfixes +to previous release branches: + +Repo/Branch | PR +------ | ------ +some_other_PR | [link]() + + +## Todos +- [ ] Tested and working on development environment +- [ ] Unit tests (if appropriate) +- [ ] Added/Updated all related documentation. Add [link]() if different from this PR +- [ ] DevOps Support needed e.g. create Runscope API test if new endpoint added or + update deployment docs. Create a ticket and add [link]() + +## Deployment Notes +Notes about how to deploy this work. For example, running a migration against the production DB. + +## How to QA +Outline the steps to test or reproduce the PR here. + +## Impacted Areas in Application +List general components of the application that this PR will affect: +- Scale +- Performance +- Security etc. \ No newline at end of file diff --git a/.gitignore b/.gitignore index 20d71296..c58d0db8 100644 --- a/.gitignore +++ b/.gitignore @@ -77,7 +77,6 @@ ntools/one-m/ansible/hosts ntools/one-m/ansible/ansible.cfg # Just in time documentation -docs/server/source/schema docs/server/source/http-samples # Terraform state files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b8fc9c9..145efe95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -145,6 +145,20 @@ Once you accept and submit the CLA, we'll email you with further instructions. ( Someone will then merge your branch or suggest changes. If we suggest changes, you won't have to open a new pull request, you can just push new code to the same branch (on `origin`) as you did before creating the pull request. +### Pull Request Guidelines + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.5, and pass the flake8 check. + Check https://travis-ci.org/bigchaindb/bigchaindb-driver/pull_requests + and make sure that the tests pass for all supported Python versions. +4. Follow the pull request template while creating new PRs, the template will + be visible to you when you create a new pull request. + ### Tip: Upgrading All BigchainDB Dependencies Over time, your versions of the Python packages used by BigchainDB will get out of date. You can upgrade them using: diff --git a/README.md b/README.md index c04a6466..7bcde402 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ BigchainDB is a scalable blockchain database. [The whitepaper](https://www.bigch ## Get Started with BigchainDB Server ### [Quickstart](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html) -### [Set Up & Run a Dev/Test Node](https://docs.bigchaindb.com/projects/server/en/latest/dev-and-test/setup-run-node.html) +### [Set Up & Run a Dev/Test Node](https://docs.bigchaindb.com/projects/server/en/latest/dev-and-test/index.html) ### [Run BigchainDB Server with Docker](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-docker.html) ### [Run BigchainDB Server with Vagrant](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-vagrant.html) ### [Run BigchainDB Server with Ansible](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-ansible.html) diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md index 3c8451b0..4b2cf873 100644 --- a/bigchaindb/common/schema/README.md +++ b/bigchaindb/common/schema/README.md @@ -3,11 +3,12 @@ This directory contains the schemas for the different JSON documents BigchainDB uses. The aim is to provide: - - a strict definition/documentation of the data structures used in BigchainDB - - a language independent tool to validate the structure of incoming/outcoming - data (there are several ready to use - [implementations](http://json-schema.org/implementations.html) written in - different languages) + +- a strict definition of the data structures used in BigchainDB +- a language independent tool to validate the structure of incoming/outcoming + data (there are several ready to use + [implementations](http://json-schema.org/implementations.html) written in + different languages) ## Learn about JSON Schema diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index fc048e0b..b2e8129a 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -13,24 +13,11 @@ from bigchaindb.common.exceptions import SchemaValidationError logger = logging.getLogger(__name__) -def drop_schema_descriptions(node): - """ Drop descriptions from schema, since they clutter log output """ - if 'description' in node: - del node['description'] - for n in node.get('properties', {}).values(): - drop_schema_descriptions(n) - for n in node.get('definitions', {}).values(): - drop_schema_descriptions(n) - for n in node.get('anyOf', []): - drop_schema_descriptions(n) - - def _load_schema(name): """ Load a schema from disk """ path = os.path.join(os.path.dirname(__file__), name + '.yaml') with open(path) as handle: schema = yaml.safe_load(handle) - drop_schema_descriptions(schema) fast_schema = rapidjson_schema.loads(rapidjson.dumps(schema)) return path, (schema, fast_schema) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index fbab5eb1..33f6a1f8 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -4,8 +4,6 @@ id: "http://www.bigchaindb.com/schema/transaction.json" type: object additionalProperties: false title: Transaction Schema -description: | - A transaction represents the creation or transfer of assets in BigchainDB. required: - id - inputs @@ -17,48 +15,24 @@ required: properties: id: "$ref": "#/definitions/sha3_hexdigest" - description: | - A sha3 digest of the transaction. The ID is calculated by removing all - derived hashes and signatures from the transaction, serializing it to - JSON with keys in sorted order and then hashing the resulting string - with sha3. operation: "$ref": "#/definitions/operation" asset: "$ref": "#/definitions/asset" - description: | - Description of the asset being transacted. - - See: `Asset`_. inputs: type: array title: "Transaction inputs" - description: | - Array of the inputs of a transaction. - - See: Input_. items: "$ref": "#/definitions/input" outputs: type: array - description: | - Array of outputs provided by this transaction. - - See: Output_. items: "$ref": "#/definitions/output" metadata: "$ref": "#/definitions/metadata" - description: | - User provided transaction metadata. This field may be ``null`` or may - contain an id and an object with freeform metadata. - - See: `Metadata`_. version: type: string pattern: "^1\\.0$" - description: | - BigchainDB transaction schema version. definitions: offset: type: integer @@ -78,53 +52,25 @@ definitions: uuid4: pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" type: string - description: | - A `UUID `_ - of type 4 (random). operation: type: string - description: | - Type of the transaction: - - A ``CREATE`` transaction creates an asset in BigchainDB. This - transaction has outputs but no inputs, so a dummy input is created. - - A ``TRANSFER`` transaction transfers ownership of an asset, by providing - an input that meets the conditions of an earlier transaction's outputs. - - A ``GENESIS`` transaction is a special case transaction used as the - sole member of the first block in a BigchainDB ledger. enum: - CREATE - TRANSFER - GENESIS asset: type: object - description: | - Description of the asset being transacted. In the case of a ``TRANSFER`` - transaction, this field contains only the ID of asset. In the case - of a ``CREATE`` transaction, this field contains only the user-defined - payload. additionalProperties: false properties: id: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID of the transaction that created the asset. data: - description: | - User provided metadata associated with the asset. May also be ``null``. anyOf: - type: object additionalProperties: true - type: 'null' output: type: object - description: | - A transaction output. Describes the quantity of an asset and the - requirements that must be met to spend the output. - - See also: Input_. additionalProperties: false required: - amount @@ -134,15 +80,7 @@ definitions: amount: type: string pattern: "^[0-9]{1,20}$" - description: | - Integral amount of the asset represented by this output. - In the case of a non divisible asset, this will always be 1. condition: - description: | - Describes the condition that needs to be met to spend the output. Has the properties: - - - **details**: Details of the condition. - - **uri**: Condition encoded as an ASCII string. type: object additionalProperties: false required: @@ -158,13 +96,8 @@ definitions: subtypes=ed25519-sha-256(&)?){2,3}$" public_keys: "$ref": "#/definitions/public_keys" - description: | - List of public keys associated with the conditions on an output. input: type: "object" - description: - An input spends a previous output, by providing one or more fulfillments - that fulfill the conditions of the previous output. additionalProperties: false required: - owners_before @@ -172,13 +105,7 @@ definitions: properties: owners_before: "$ref": "#/definitions/public_keys" - description: | - List of public keys of the previous owners of the asset. fulfillment: - description: | - Fulfillment of an `Output.condition`_, or, put a different way, a payload - that satisfies the condition of a previous output to prove that the - creator(s) of this transaction have control over the listed asset. anyOf: - type: string pattern: "^[a-zA-Z0-9_-]*$" @@ -186,8 +113,6 @@ definitions: fulfills: anyOf: - type: 'object' - description: | - Reference to the output that is being spent. additionalProperties: false required: - output_index @@ -195,26 +120,16 @@ definitions: properties: output_index: "$ref": "#/definitions/offset" - description: | - Index of the output containing the condition being fulfilled transaction_id: "$ref": "#/definitions/sha3_hexdigest" - description: | - Transaction ID containing the output to spend - type: 'null' metadata: anyOf: - type: object - description: | - User provided transaction metadata. This field may be ``null`` or may - contain an non empty object with freeform metadata. additionalProperties: true minProperties: 1 - type: 'null' condition_details: - description: | - Details needed to reconstruct the condition associated with an output. - Currently, BigchainDB only supports ed25519 and threshold condition types. anyOf: - type: object additionalProperties: false diff --git a/bigchaindb/common/schema/transaction_transfer.yaml b/bigchaindb/common/schema/transaction_transfer.yaml index b8b79696..538ec5e6 100644 --- a/bigchaindb/common/schema/transaction_transfer.yaml +++ b/bigchaindb/common/schema/transaction_transfer.yaml @@ -10,8 +10,6 @@ properties: properties: id: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID of the transaction that created the asset. required: - id inputs: diff --git a/bigchaindb/common/schema/vote.yaml b/bigchaindb/common/schema/vote.yaml index 49e5ae98..8f16562f 100644 --- a/bigchaindb/common/schema/vote.yaml +++ b/bigchaindb/common/schema/vote.yaml @@ -4,13 +4,6 @@ id: "http://www.bigchaindb.com/schema/vote.json" type: object additionalProperties: false title: Vote Schema -description: | - A Vote is an endorsement of a Block (identified by a hash) by - a node (identified by a public key). - - The outer Vote object contains the details of the vote being made - as well as the signature and identifying information of the node - passing the vote. required: - node_pubkey - signature @@ -19,18 +12,12 @@ properties: node_pubkey: type: "string" pattern: "[1-9a-zA-Z^OIl]{43,44}" - description: | - Ed25519 public key identifying the voting node. signature: type: "string" pattern: "[1-9a-zA-Z^OIl]{86,88}" - description: - Ed25519 signature of the `Vote Details`_ object. vote: type: "object" additionalProperties: false - description: | - `Vote Details`_ to be signed. required: - invalid_reason - is_block_valid @@ -40,33 +27,17 @@ properties: properties: previous_block: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID (SHA3 hash) of the block that precedes the block being voted on. - The notion of a "previous" block is subject to vote. voting_for_block: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID (SHA3 hash) of the block being voted on. is_block_valid: type: "boolean" - description: | - This field is ``true`` if the block was deemed valid by the node. invalid_reason: anyOf: - type: "string" - description: | - Reason the block is voted invalid, or ``null``. - - .. container:: notice - - **Note**: The invalid_reason was not being used and may be dropped in a future version of BigchainDB. See Issue `#217 `_ on GitHub. - type: "null" timestamp: type: "string" pattern: "[0-9]{10}" - description: | - Unix timestamp that the vote was created by the node, according - to the system time of the node. definitions: sha3_hexdigest: pattern: "[0-9a-f]{64}" diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py deleted file mode 100644 index c94fe3a9..00000000 --- a/docs/server/generate_schema_documentation.py +++ /dev/null @@ -1,241 +0,0 @@ -""" Script to render transaction schema into .rst document """ - -from collections import OrderedDict -import os.path - -import yaml - -from bigchaindb.common.schema import TX_SCHEMA_PATH, VOTE_SCHEMA_PATH - - -TPL_PROP = """\ -%(title)s -%(underline)s - -**type:** %(type)s - -%(description)s -""" - - -TPL_STYLES = """ -.. raw:: html - - -""" - - -TPL_TRANSACTION = TPL_STYLES + """\ -.. This file was auto generated by %(file)s - -================== -Transaction Schema -================== - -* `Transaction`_ - -* Input_ - -* Output_ - -* Asset_ - -* Metadata_ - - -Transaction ------------ - -%(transaction)s - -Input ------ - -%(input)s - -Output ------- - -%(output)s - -Asset ------ - -%(asset)s - -Metadata --------- - -%(metadata)s -""" - - -def generate_transaction_docs(): - schema = load_schema(TX_SCHEMA_PATH) - defs = schema['definitions'] - - doc = TPL_TRANSACTION % { - 'transaction': render_section('Transaction', schema), - 'output': render_section('Output', defs['output']), - 'input': render_section('Input', defs['input']), - 'asset': render_section('Asset', defs['asset']), - 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), - 'container': 'transaction-schema', - 'file': os.path.basename(__file__), - } - - write_schema_doc('transaction', doc) - - -TPL_VOTE = TPL_STYLES + """\ -.. This file was auto generated by %(file)s - -=========== -Vote Schema -=========== - -Vote ----- - -%(vote)s - -Vote Details ------------- - -%(vote_details)s - -""" - - -def generate_vote_docs(): - schema = load_schema(VOTE_SCHEMA_PATH) - - doc = TPL_VOTE % { - 'vote': render_section('Vote', schema), - 'vote_details': render_section('Vote', schema['properties']['vote']), - 'container': 'vote-schema', - 'file': os.path.basename(__file__), - } - - write_schema_doc('vote', doc) - - -def ordered_load_yaml(path): - """ Custom YAML loader to preserve key order """ - class OrderedLoader(yaml.SafeLoader): - pass - - def construct_mapping(loader, node): - loader.flatten_mapping(node) - return OrderedDict(loader.construct_pairs(node)) - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping) - with open(path) as handle: - return yaml.load(handle, OrderedLoader) - - -def load_schema(path): - global DEFS - schema = ordered_load_yaml(path) - DEFS = schema['definitions'] - return schema - - -def write_schema_doc(name, doc): - # Check base path exists - base_path = os.path.join(os.path.dirname(__file__), 'source/schema') - if not os.path.exists(base_path): - os.makedirs(base_path) - # Write doc - path = os.path.join(base_path, '%s.rst' % name) - with open(path, 'w') as handle: - handle.write(doc) - - -def render_section(section_name, obj): - """ Render a domain object and it's properties """ - out = [obj['description']] - for name, prop in obj.get('properties', {}).items(): - try: - title = '%s.%s' % (section_name, name) - out += [TPL_PROP % { - 'title': title, - 'underline': '^' * len(title), - 'description': property_description(prop), - 'type': property_type(prop), - }] - except Exception as exc: - raise ValueError('Error rendering property: %s' % name, exc) - return '\n\n'.join(out + ['']) - - -def property_description(prop): - """ Get description of property """ - if 'description' in prop: - return prop['description'] - if '$ref' in prop: - return property_description(resolve_ref(prop['$ref'])) - if 'anyOf' in prop: - return property_description(prop['anyOf'][0]) - raise KeyError('description') - - -def property_type(prop): - """ Resolve a string representing the type of a property """ - if 'type' in prop: - if prop['type'] == 'array': - return 'array (%s)' % property_type(prop['items']) - return prop['type'] - if 'anyOf' in prop: - return ' or '.join(property_type(p) for p in prop['anyOf']) - if '$ref' in prop: - return property_type(resolve_ref(prop['$ref'])) - raise ValueError('Could not resolve property type') - - -DEFINITION_BASE_PATH = '#/definitions/' - - -def resolve_ref(ref): - """ Resolve definition reference """ - assert ref.startswith(DEFINITION_BASE_PATH) - return DEFS[ref[len(DEFINITION_BASE_PATH):]] - - -def main(): - """ Main function """ - generate_transaction_docs() - generate_vote_docs() - - -def setup(*_): - """ Fool sphinx into think it's an extension muahaha """ - main() - - -if __name__ == '__main__': - main() diff --git a/docs/server/source/appendices/docker-on-mac.md b/docs/server/source/appendices/docker-on-mac.md deleted file mode 100644 index 7f87540f..00000000 --- a/docs/server/source/appendices/docker-on-mac.md +++ /dev/null @@ -1,101 +0,0 @@ -# Run BigchainDB with Docker On Mac - -**NOT for Production Use** - -Those developing on Mac can follow this document to run BigchainDB in docker -containers for a quick dev setup. -Running BigchainDB on Mac (Docker or otherwise) is not officially supported. - -Support is very much limited as there are certain things that work differently -in Docker for Mac than Docker for other platforms. -Also, we do not use mac for our development and testing. :) - -This page may not be up to date with various settings and docker updates at -all the times. - -These steps work as of this writing (2017.Mar.09) and might break in the -future with updates to Docker for mac. -Community contribution to make BigchainDB run on Docker for Mac will always be -welcome. - - -## Prerequisite - -Install Docker for Mac. - -## (Optional) For a clean start - -1. Stop all BigchainDB and RethinkDB/MongoDB containers. -2. Delete all BigchainDB docker images. -3. Delete the ~/bigchaindb_docker folder. - - -## Pull the images - -Pull the bigchaindb and other required docker images from docker hub. - -```text -docker pull bigchaindb/bigchaindb:master -docker pull [rethinkdb:2.3|mongo:3.4.1] -``` - -## Create the BigchainDB configuration file on Mac -```text -docker run \ - --rm \ - --volume $HOME/bigchaindb_docker:/data \ - bigchaindb/bigchaindb:master \ - -y configure \ - [mongodb|rethinkdb] -``` - -To ensure that BigchainDB connects to the backend database bound to the virtual -interface `172.17.0.1`, you must edit the BigchainDB configuration file -(`~/bigchaindb_docker/.bigchaindb`) and change database.host from `localhost` -to `172.17.0.1`. - - -## Run the backend database on Mac - -From v0.9 onwards, you can run RethinkDB or MongoDB. - -We use the virtual interface created by the Docker daemon to allow -communication between the BigchainDB and database containers. -It has an IP address of 172.17.0.1 by default. - -You can also use docker host networking or bind to your primary (eth) -interface, if needed. - -### For RethinkDB backend -```text -docker run \ - --name=rethinkdb \ - --publish=28015:28015 \ - --publish=8080:8080 \ - --restart=always \ - --volume $HOME/bigchaindb_docker:/data \ - rethinkdb:2.3 -``` - -### For MongoDB backend -```text -docker run \ - --name=mongodb \ - --publish=27017:27017 \ - --restart=always \ - --volume=$HOME/bigchaindb_docker/db:/data/db \ - --volume=$HOME/bigchaindb_docker/configdb:/data/configdb \ - mongo:3.4.1 --replSet=bigchain-rs -``` - -### Run BigchainDB on Mac -```text -docker run \ - --name=bigchaindb \ - --publish=9984:9984 \ - --restart=always \ - --volume=$HOME/bigchaindb_docker:/data \ - bigchaindb/bigchaindb \ - start -``` - diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 7bee77df..6f560458 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -10,7 +10,6 @@ Appendices install-os-level-deps install-latest-pip run-with-docker - docker-on-mac json-serialization cryptography the-Bigchain-class @@ -28,4 +27,6 @@ Appendices licenses install-with-lxd run-with-vagrant - run-with-ansible \ No newline at end of file + run-with-ansible + tx-yaml-files + vote-yaml diff --git a/docs/server/source/appendices/run-with-ansible.md b/docs/server/source/appendices/run-with-ansible.md index 70e4b595..b1ec6769 100644 --- a/docs/server/source/appendices/run-with-ansible.md +++ b/docs/server/source/appendices/run-with-ansible.md @@ -2,8 +2,8 @@ **NOT for Production Use** -You can use the following instructions to deploy a BigchainDB node for -dev/test using Ansible. Ansible will setup a BigchainDB node along with +You can use the following instructions to deploy a single or multi node +BigchainDB setup for dev/test using Ansible. Ansible will setup BigchainDB node(s) along with [Docker](https://www.docker.com/), [Docker Compose](https://docs.docker.com/compose/), [MongoDB](https://www.mongodb.com/), [BigchainDB Python driver](https://docs.bigchaindb.com/projects/py-driver/en/latest/). @@ -12,6 +12,10 @@ Currently, this workflow is only supported for the following distributions: - CentOS >= 7 - Fedora >= 24 +## Minimum Requirements | Ansible +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 ## Clone the BigchainDB repository | Ansible ```text $ git clone https://github.com/bigchaindb/bigchaindb.git @@ -20,54 +24,93 @@ $ git clone https://github.com/bigchaindb/bigchaindb.git ## Install dependencies | Ansible - [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) -You can also install `ansible` and other dependecies, if any, using the `boostrap.sh` script +You can also install `ansible` and other dependencies, if any, using the `boostrap.sh` script inside the BigchainDB repository. Navigate to `bigchaindb/pkg/scripts` and run the `bootstrap.sh` script to install the dependecies for your OS. The script also checks if the OS you are running is compatible with the supported versions. +**Note**: `bootstrap.sh` only supports Ubuntu >= 16.04, CentOS >= 7 and Fedora >=24. + ```text $ cd bigchaindb/pkg/scripts/ $ sudo ./bootstrap.sh ``` -### Local Setup | Ansible -You can safely run the `quickstart` playbook now and everything will be taken care of by `ansible` on your host. `quickstart` playbook only supports deployment on your dev/local host. To run the playbook please navigate to the ansible directory inside the BigchainDB repository and run the `quickstart` playbook. +### BigchainDB Setup Configuration(s) | Ansible +#### Local Setup | Ansible +You can run the Ansible playbook `bdb-deploy.yml` on your local dev machine and set up the BigchainDB node where +BigchainDB can be run as a process or inside a Docker container(s) depending on your configuration. +Before, running the playbook locally, you need to update the `hosts` and `bdb-config.yml` configuration, which will notify Ansible that we need to run the play locally. + +##### Update Hosts | Local +Navigate to `bigchaindb/pkg/configuration/hosts` inside the BigchainDB repository. ```text -$ cd bigchaindb/pkg/ansible/ - -# All the services will be deployed as processes -$ sudo ansible-playbook quickstart.yml -c local - -OR - -# To deploy all services inside docker containers -$ sudo ansible-playbook quickstart.yml --extra-vars "with_docker=true" -c local +$ cd bigchaindb/pkg/configuration/hosts ``` -After successfull execution of the playbook, you can verify that BigchainDB docker/process is running. +Edit `all` configuration file: +```text +# Delete any existing configuration in this file and insert +# Hostname of dev machine + ansible_connection=local +``` +##### Update Configuration | Local +Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/vars/bdb-config.yml +``` -Verify BigchainDB process: +Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): +```text +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 # Only needed if `deploy_docker` is true +bdb_hosts: + - name: "" # Hostname of dev machine +``` +**Note**: You can also orchestrate a multi-node BigchainDB cluster on a local dev host using Docker containers. +Here is a sample `bdb-config.yml` +```text +--- +deploy_docker: true #[true, false] +docker_cluster_size: 3 +bdb_hosts: + - name: "" +``` + +### BigchainDB Setup | Ansible +Now, You can safely run the `bdb-deploy.yml` playbook and everything will be taken care of by `Ansible`. To run the playbook please navigate to the `bigchaindb/pkg/configuration` directory inside the BigchainDB repository and run the `bdb-deploy.yml` playbook. + +```text +$ cd bigchaindb/pkg/configuration/ + +$ sudo ansible-playbook bdb-deploy.yml -i hosts/all +``` + +After successful execution of the playbook, you can verify that BigchainDB docker(s)/process(es) is(are) running. + +Verify BigchainDB process(es): ```text $ ps -ef | grep bigchaindb ``` OR -Verify BigchainDB Docker: +Verify BigchainDB Docker(s): ```text $ docker ps | grep bigchaindb ``` -The playbook also installs the BigchainDB Python Driver, +The playbook also installs the BigchainDB Python Driver, so you can use it to make transactions and verify the functionality of your BigchainDB node. See the [BigchainDB Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) for details on how to use it. -Note 1: The `bdb_root_url` can be be one of the following: +**Note**: The `bdb_root_url` can be be one of the following: ```text # BigchainDB is running as a process bdb_root_url = http://:9984 @@ -78,4 +121,47 @@ OR bdb_root_url = http://: ``` -Note 2: BigchainDB has [other drivers as well](../drivers-clients/index.html). +**Note**: BigchainDB has [other drivers as well](../drivers-clients/index.html). + +### Experimental: Running Ansible a Remote Dev/Host +#### Remote Setup | Ansible +You can also run the Ansible playbook `bdb-deploy.yml` on remote machine(s) and set up the BigchainDB node where +BigchainDB can run as a process or inside a Docker container(s) depending on your configuration. + +Before, running the playbook on a remote host, you need to update the `hosts` and `bdb-config.yml` configuration, which will notify Ansible that we need to +run the play on a remote host. + +##### Update Hosts | Remote +Navigate to `bigchaindb/pkg/configuration/hosts` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/hosts +``` + +Edit `all` configuration file: +```text +# Delete any existing configuration in this file and insert + ansible_ssh_user= ansible_sudo_pass= +``` + +**Note**: You can add multiple hosts to the `all` configuration file. Root password is needed because ansible +will run some tasks that require root permissions. + +**Note**: You can also use other methods to get inside the remote machines instead of password based SSH. For other methods +please consult [Ansible Documentation](http://docs.ansible.com/ansible/latest/intro_getting_started.html). + +##### Update Configuration | Remote +Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/vars/bdb-config.yml +``` + +Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): +```text +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 # Only needed if `deploy_docker` is true +bdb_hosts: + - name: "" +``` + +After, the configuration of remote hosts, [run the Ansible playbook and verify your deployment](#bigchaindb-setup-ansible). diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index d80f58b1..b75c9300 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -6,9 +6,12 @@ For those who like using Docker and wish to experiment with BigchainDB in non-production environments, we currently maintain a Docker image and a `Dockerfile` that can be used to build an image for `bigchaindb`. +## Prerequisite(s) +- [Docker](https://docs.docker.com/engine/installation/) + ## Pull and Run the Image from Docker Hub -Assuming you have Docker installed, you would proceed as follows. +With Docker installed, you can proceed as follows. In a terminal shell, pull the latest version of the BigchainDB Docker image using: ```text @@ -26,6 +29,7 @@ docker run \ --rm \ --tty \ --volume $HOME/bigchaindb_docker:/data \ + --env BIGCHAINDB_DATABASE_HOST=172.17.0.1 \ bigchaindb/bigchaindb \ -y configure \ [mongodb|rethinkdb] @@ -46,24 +50,18 @@ Let's analyze that command: this allows us to have the data persisted on the host machine, you can read more in the [official Docker documentation](https://docs.docker.com/engine/tutorials/dockervolumes) +* `--env BIGCHAINDB_DATABASE_HOST=172.17.0.1`, `172.17.0.1` is the default `docker0` bridge +IP address, for fresh Docker installations. It is used for the communication between BigchainDB and database +containers. * `bigchaindb/bigchaindb` the image to use. All the options after the container name are passed on to the entrypoint inside the container. * `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values * `mongodb` or `rethinkdb` specifies the database backend to use with bigchaindb -To ensure that BigchainDB connects to the backend database bound to the virtual -interface `172.17.0.1`, you must edit the BigchainDB configuration file -(`~/bigchaindb_docker/.bigchaindb`) and change database.host from `localhost` -to `172.17.0.1`. - ### Run the backend database From v0.9 onwards, you can run either RethinkDB or MongoDB. -We use the virtual interface created by the Docker daemon to allow -communication between the BigchainDB and database containers. -It has an IP address of 172.17.0.1 by default. - You can also use docker host networking or bind to your primary (eth) interface, if needed. @@ -73,8 +71,8 @@ You can also use docker host networking or bind to your primary (eth) docker run \ --detach \ --name=rethinkdb \ - --publish=172.17.0.1:28015:28015 \ - --publish=172.17.0.1:58080:8080 \ + --publish=28015:28015 \ + --publish=58080:8080 \ --restart=always \ --volume $HOME/bigchaindb_docker:/data \ rethinkdb:2.3 @@ -102,11 +100,11 @@ group. docker run \ --detach \ --name=mongodb \ - --publish=172.17.0.1:27017:27017 \ + --publish=27017:27017 \ --restart=always \ --volume=$HOME/mongodb_docker/db:/data/db \ --volume=$HOME/mongodb_docker/configdb:/data/configdb \ - mongo:3.4.1 --replSet=bigchain-rs + mongo:3.4.9 --replSet=bigchain-rs ``` ### Run BigchainDB diff --git a/docs/server/source/appendices/run-with-vagrant.md b/docs/server/source/appendices/run-with-vagrant.md index c396017a..8bde0a38 100644 --- a/docs/server/source/appendices/run-with-vagrant.md +++ b/docs/server/source/appendices/run-with-vagrant.md @@ -2,10 +2,10 @@ **NOT for Production Use** -You can use the following instructions to deploy a BigchainDB node -for dev/test using Vagrant. Vagrant will setup a BigchainDB node with -all the dependencies along with MongoDB, BigchainDB Python driver. You -can also tweak the following configurations for the BigchainDB node. +You can use the following instructions to deploy a single or multi node +BigchainDB setup for dev/test using Vagrant. Vagrant will set up the BigchainDB node(s) +with all the dependencies along with MongoDB and BigchainDB Python driver. You +can also tweak the following configurations for the BigchainDB node(s). - Vagrant Box - Currently, we support the following boxes: - `ubuntu/xenial64 # >=16.04` @@ -19,15 +19,21 @@ can also tweak the following configurations for the BigchainDB node. - Network Type - Currently, only `private_network` is supported. - IP Address -- Setup type - - `quickstart` - Deploy node with Docker - Deploy all the services in Docker containers or as processes. +- Number of BigchainDB nodes + - If you want to deploy the services inside Docker containers, you + can specify number of member(s) in the BigchainDB cluster. - Upstart Script - Vagrant Provider - Virtualbox - VMware +## Minimum Requirements | Vagrant +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 + ## Install dependencies | Vagrant 1. [VirtualBox](https://www.virtualbox.org/wiki/Downloads) >= 5.0.0 2. [Vagrant](https://www.vagrantup.com/downloads.html) >= 1.16.0 @@ -38,86 +44,108 @@ $ git clone https://github.com/bigchaindb/bigchaindb.git ``` ## Configuration | Vagrant -Navigate to `bigchaindb/pkg/config/` inside the repository. +Navigate to `bigchaindb/pkg/configuration/vars/` inside the BigchainDB repository. ```text -$ cd bigchaindb/pkg/config/ +$ cd bigchaindb/pkg/configuration/vars/ ``` -Edit the `bdb-config.yaml` as per your requirements. Sample `bdb-config.yaml`: +Edit `bdb-config.yml` as per your requirements. Sample `bdb-config.yml`: ```text --- -- name: "bdb-node-01" - box: - name: "ubuntu/xenial64" - ram: "2048" - vcpus: "2" - setup_type: "quickstart" - deploy_docker: false - network: - ip: "10.20.30.40" - type: "private_network" - upstart: "/bigchaindb/scripts/bootstrap.sh" +deploy_docker: false #[true, false] +docker_cluster_size: 1 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.40" + type: "private_network" ``` -**Note**: You can spawn multiple instances as well using `bdb-config.yaml`. Here is a sample `bdb-config.yaml`: +**Note**: You can spawn multiple instances to orchestrate a multi-node BigchainDB cluster. +Here is a sample `bdb-config.yml`: ```text --- -- name: "bdb-node-01" - box: - name: "ubuntu/xenial64" - ram: "2048" - vcpus: "2" - setup_type: "quickstart" - deploy_docker: false - network: - ip: "10.20.30.40" - type: "private_network" - upstart: "/bigchaindb/scripts/bootstrap.sh" -- name: "bdb-node-02" - box: - name: "ubuntu/xenial64" - ram: "4096" - vcpus: "3" - setup_type: "quickstart" - deploy_docker: false - network: - ip: "10.20.30.50" - type: "private_network" - upstart: "/bigchaindb/scripts/bootstrap.sh" +deploy_docker: false #[true, false] +docker_cluster_size: 1 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.40" + type: "private_network" + - name: "bdb-node-02" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.50" + type: "private_network" +``` +**Note**: You can also orchestrate a multi-node BigchainDB cluster on a single dev host using Docker containers. +Here is a sample `bdb-config.yml` +```text +--- +deploy_docker: true #[true, false] +docker_cluster_size: 3 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "8192" + vcpus: "4" + network: + ip: "10.20.30.40" + type: "private_network" +``` +The above mentioned configuration will deploy a 3 node BigchainDB cluster with Docker containers +on your specified host. + +## BigchainDB Setup | Vagrant + +**Note**: There are some vagrant plugins required for the installation, +user will be prompted to install them if they are not present. To install +the required plugins, run the following command: + +```text +$ vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts ``` - -## Local Setup | Vagrant -To bring up the BigchainDB node, run the following command: +To bring up the BigchainDB node(s), run the following command: ```text $ vagrant up ``` -*Note*: There are some vagrant plugins required for the installation, user will be prompted to install them if they are not present. Instructions to install the plugins can be extracted from the message. - -```text -$ vagrant plugin install -``` - -After successfull execution of Vagrant, you can log in to your fresh BigchainDB node. +After successful execution of Vagrant, you can log in to your fresh BigchainDB node. ```text $ vagrant ssh ``` ## Make your first transaction -Once you are inside the BigchainDB node, you can verify that BigchainDB docker/process is running. +Once you are inside the BigchainDB node, you can verify that BigchainDB +docker(s)/process(es) is(are) running. -Verify BigchainDB process: +Verify BigchainDB process(es): ```text $ ps -ef | grep bigchaindb ``` OR -Verify BigchainDB Docker: +Verify BigchainDB Docker(s): ```text $ docker ps | grep bigchaindb ``` diff --git a/docs/server/source/appendices/tx-yaml-files.rst b/docs/server/source/appendices/tx-yaml-files.rst new file mode 100644 index 00000000..f9a51685 --- /dev/null +++ b/docs/server/source/appendices/tx-yaml-files.rst @@ -0,0 +1,37 @@ +The Transaction Schema Files +============================ + +BigchainDB checks all :ref:`transactions ` +(JSON documents) against a formal schema +defined in some JSON Schema files named +transaction.yaml, +transaction_create.yaml and +transaction_transfer.yaml. +The contents of those files are copied below. +To understand those contents +(i.e. JSON Schema), check out +`"Understanding JSON Schema" +`_ +by Michael Droettboom or +`json-schema.org `_. + + +transaction.yaml +---------------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/transaction.yaml + :language: yaml + + +transaction_create.yaml +----------------------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/transaction_create.yaml + :language: yaml + + +transaction_transfer.yaml +------------------------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/transaction_transfer.yaml + :language: yaml diff --git a/docs/server/source/appendices/vote-yaml.rst b/docs/server/source/appendices/vote-yaml.rst new file mode 100644 index 00000000..6613827e --- /dev/null +++ b/docs/server/source/appendices/vote-yaml.rst @@ -0,0 +1,20 @@ +The Vote Schema File +==================== + +BigchainDB checks all :ref:`votes ` +(JSON documents) against a formal schema +defined in a JSON Schema file named vote.yaml. +The contents of that file are copied below. +To understand those contents +(i.e. JSON Schema), check out +`"Understanding JSON Schema" +`_ +by Michael Droettboom or +`json-schema.org `_. + + +vote.yaml +--------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/vote.yaml + :language: yaml diff --git a/docs/server/source/conf.py b/docs/server/source/conf.py index 756a8d13..e272baf4 100644 --- a/docs/server/source/conf.py +++ b/docs/server/source/conf.py @@ -51,7 +51,6 @@ extensions = [ 'sphinx.ext.autosectionlabel', # Below are actually build steps made to look like sphinx extensions. # It was the easiest way to get it running with ReadTheDocs. - 'generate_schema_documentation', 'generate_http_server_api_documentation', ] diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index 8b184261..36b7204e 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -1,36 +1,90 @@ The Block Model =============== -A block has the following structure: +A block is a JSON object with a particular schema, +as outlined in this page. +A block must contain the following JSON keys +(also called names or fields): .. code-block:: json { - "id": "", + "id": "", "block": { - "timestamp": "", - "transactions": [""], - "node_pubkey": "", - "voters": [""] + "timestamp": "", + "transactions": [""], + "node_pubkey": "", + "voters": [""] }, - "signature": "" + "signature": "" } -- ``id``: The :ref:`hash ` of the serialized inner ``block`` (i.e. the ``timestamp``, ``transactions``, ``node_pubkey``, and ``voters``). It's used as a unique index in the database backend (e.g. RethinkDB or MongoDB). +The JSON Keys in a Block +------------------------ -- ``block``: - - ``timestamp``: The Unix time when the block was created. It's provided by the node that created the block. - - ``transactions``: A list of the transactions included in the block. - - ``node_pubkey``: The public key of the node that created the block. - - ``voters``: A list of the public keys of all cluster nodes at the time the block was created. - It's the list of nodes which can cast a vote on this block. - This list can change from block to block, as nodes join and leave the cluster. +**id** -- ``signature``: :ref:`Cryptographic signature ` of the block by the node that created the block (i.e. the node with public key ``node_pubkey``). To generate the signature, the node signs the serialized inner ``block`` (the same thing that was hashed to determine the ``id``) using the private key corresponding to ``node_pubkey``. +The transaction ID and also the SHA3-256 hash +of the inner ``block`` object, loosely speaking. +It's a string. +To compute it, 1) construct an :term:`associative array` ``d`` containing +``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, +``block.voters``, and their values. 2) compute ``id = hash_of_aa(d)``. +There's pseudocode for the ``hash_of_aa()`` function +in the `IPDB Protocol documentation page about cryptographic hashes +`_. +The result (``id``) is a string: the block ID. +An example is ``"b60adf655932bf47ef58c0bfb2dd276d4795b94346b36cbb477e10d7eb02cea8"`` -Working with Blocks -------------------- +**block.timestamp** -There's a **Block** class for creating and working with Block objects; look in `/bigchaindb/models.py `_. (The link is to the latest version on the master branch on GitHub.) +The `Unix time `_ +when the block was created, according to the node which created it. +It's a string representation of an integer. +An example is ``"1507294217"``. + + +**block.transactions** + +A list of the :ref:`transactions ` included in the block. +(Each transaction is a JSON object.) + + +**block.node_pubkey** + +The public key of the node that created the block. +It's a string. +See the `IPDB Protocol documentation page about cryptographic keys & signatures +`_. + + +**block.voters** + +A list of the public keys of all cluster nodes at the time the block was created. +It's a list of strings. +This list can change from block to block, as nodes join and leave the cluster. + + +**signature** + +The cryptographic signature of the inner ``block`` +by the node that created the block +(i.e. the node with public key ``node_pubkey``). +To compute that: + +#. Construct an :term:`associative array` ``d`` containing the contents + of the inner ``block`` + (i.e. ``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, + ``block.voters``, and their values). +#. Compute ``signature = sig_of_aa(d, private_key)``, + where ``private_key`` is the node's private key + (i.e. ``node_pubkey`` and ``private_key`` are a key pair). There's pseudocode + for the ``sig_of_aa()`` function + on `the IPDB Protocol documentation page about cryptographic keys and signatures + `_. + +.. note:: + + The ``d_bytes`` computed when computing the block ID will be the *same* as the ``d_bytes`` computed when computing the block signature. This can be used to avoid redundant calculations. diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index 4e246bfe..52214421 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -26,7 +26,6 @@ An input has the following structure: You can think of the ``fulfills`` object as a pointer to an output on another transaction: the output that this input is spending/transferring. A CREATE transaction should have exactly one input. That input can contain one or more ``owners_before``, a ``fulfillment`` (with one signature from each of the owners-before), and the value of ``fulfills`` should be ``null``). A TRANSFER transaction should have at least one input, and the value of ``fulfills`` should not be ``null``. -See the reference on :ref:`inputs ` for more description about the meaning of each field. The ``fulfillment`` string fulfills the condition in the output that is being spent (transferred). To calculate it: @@ -62,7 +61,6 @@ An output has the following structure: The :ref:`page about conditions ` explains the contents of a ``condition``. The list of ``public_keys`` is always the "owners" of the asset at the time the transaction completed, but before the next transaction started. -See the reference on :ref:`outputs ` for more description about the meaning of each field. Note that ``amount`` must be a string (e.g. ``"7"``). In a TRANSFER transaction, the sum of the output amounts must be the same as the sum of the outputs that it transfers (i.e. the sum of the input amounts). For example, if a TRANSFER transaction has two outputs, one with ``"amount": "2"`` and one with ``"amount": "3"``, then the sum of the outputs is 5 and so the sum of the outputs-being-transferred must also be 5. diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index b4f2a4f1..631399e2 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -19,20 +19,18 @@ Here's some explanation of the contents: - **id**: The ID of the transaction and also the hash of the transaction (loosely speaking). See below for an explanation of how it's computed. It's also the database primary key. -- **version**: The version-number of :ref:`the transaction schema `. As of BigchainDB Server 1.0.0, the only allowed value is ``"1.0"``. +- **version**: The version-number of the transaction schema. As of BigchainDB Server 1.0.0, the only allowed value is ``"1.0"``. - **inputs**: List of inputs. Each input spends/transfers a previous output by satisfying/fulfilling the crypto-conditions on that output. A CREATE transaction should have exactly one input. A TRANSFER transaction should have at least one input (i.e. ≥1). - For more details, see the subsection about :ref:`inputs `. - **outputs**: List of outputs. Each output indicates the crypto-conditions which must be satisfied by anyone wishing to spend/transfer that output. It also indicates the number of shares of the asset tied to that output. - For more details, see the subsection about :ref:`outputs `. - **operation**: A string indicating what kind of transaction this is, and how it should be validated. @@ -64,3 +62,13 @@ There are example BigchainDB transactions in :ref:`the HTTP API documentation ` and `the Python Driver documentation `_. + + +The Transaction Schema +---------------------- + +BigchainDB checks all transactions (JSON documents) +against a formal schema defined in :ref:`some JSON Schema files named +transaction.yaml, +transaction_create.yaml and +transaction_transfer.yaml `. diff --git a/docs/server/source/data-models/vote-model.md b/docs/server/source/data-models/vote-model.md deleted file mode 100644 index daa66a94..00000000 --- a/docs/server/source/data-models/vote-model.md +++ /dev/null @@ -1,27 +0,0 @@ -# The Vote Model - -A vote has the following structure: - -```json -{ - "node_pubkey": "", - "vote": { - "voting_for_block": "", - "previous_block": "", - "is_block_valid": "", - "invalid_reason": null, - "timestamp": "" - }, - "signature": "" -} -``` - -**Notes** - -* Votes have no ID (or `"id"`), as far as users are concerned. (The backend database uses one internally, but it's of no concern to users and it's never reported to them via BigchainDB APIs.) - -* At the time of writing, the value of `"invalid_reason"` was always `null`. In other words, it wasn't being used. It may be used or dropped in a future version of BigchainDB. See [Issue #217](https://github.com/bigchaindb/bigchaindb/issues/217) on GitHub. - -* For more information about the vote `"timestamp"`, see [the page about timestamps in BigchainDB](https://docs.bigchaindb.com/en/latest/timestamps.html). - -* For more information about how the `"signature"` is calculated, see [the page about cryptography in BigchainDB](../appendices/cryptography.html). diff --git a/docs/server/source/data-models/vote-model.rst b/docs/server/source/data-models/vote-model.rst new file mode 100644 index 00000000..45db9680 --- /dev/null +++ b/docs/server/source/data-models/vote-model.rst @@ -0,0 +1,121 @@ +The Vote Model +============== + +A vote is a JSON object with a particular schema, +as outlined in this page. +A vote must contain the following JSON keys +(also called names or fields): + +.. code-block:: json + + { + "node_pubkey": "", + "vote": { + "voting_for_block": "", + "previous_block": "", + "is_block_valid": "", + "invalid_reason": null, + "timestamp": "" + }, + "signature": "" + } + +.. note:: + + Votes have no ID (or ``"id"``), as far as users are concerned. + The backend database may use one internally, + but it's of no concern to users and it's never reported to them via APIs. + + +The JSON Keys in a Vote +----------------------- + +**node_pubkey** + +The public key of the node which cast this vote. +It's a string. +For more information about public keys, +see the `IPDB Protocol documentation page about cryptographic keys and signatures +`_. + + +**vote.voting_for_block** + +The block ID that this vote is for. +It's a string. +For more information about block IDs, +see the page about :ref:`blocks `. + + +**vote.previous_block** + +The block ID of the block "before" the block that this vote is for, +according to the node which cast this vote. +It's a string. +(It's possible for different nodes to see different block orders.) +For more information about block IDs, +see the page about :ref:`blocks `. + + +**vote.is_block_valid** + +``true`` if the node which cast this vote considered the block in question to be valid, +and ``false`` otherwise. +Note that it's a *boolean* (i.e. ``true`` or ``false``), not a string. + + +**vote.invalid_reason** + +Always ``null``, that is, it's not being used. +It may be used or dropped in a future version. +See `bigchaindb/bigchaindb issue #217 +`_ on GitHub. + + +**vote.timestamp** + +The `Unix time `_ +when the vote was created, according to the node which created it. +It's a string representation of an integer. + + +**signature** + +The cryptographic signature of the inner ``vote`` +by the node that created the vote +(i.e. the node with public key ``node_pubkey``). +To compute that: + +#. Construct an :term:`associative array` ``d`` containing the contents of the inner ``vote`` + (i.e. ``vote.voting_for_block``, ``vote.previous_block``, ``vote.is_block_valid``, + ``vote.invalid_reason``, ``vote.timestamp``, and their values). +#. Compute ``signature = sig_of_aa(d, private_key)``, where ``private_key`` + is the node's private key (i.e. ``node_pubkey`` and ``private_key`` are a key pair). + There's pseudocode for the ``sig_of_aa()`` function + on `the IPDB Protocol documentation page about cryptographic keys and signatures + `_. + + +The Vote Schema +--------------- + +BigchainDB checks all votes (JSON documents) against a formal schema +defined in a :ref:`JSON Schema file named vote.yaml `. + + +An Example Vote +--------------- + +.. code-block:: json + + { + "node_pubkey": "3ZCsVWPAhPTqHx9wZVxp9Se54pcNeeM5mQvnozDWyDR9", + "vote": { + "voting_for_block": "11c3a3fcc9efa4fc4332a0849fc39b58e403ff37794a7d1fdfb9e7703a94a274", + "previous_block": "3dd1441018b782a50607dc4c7f83a0f0a23eb257f4b6a8d99330dfff41271e0d", + "is_block_valid": true, + "invalid_reason": null, + "timestamp": "1509977988" + }, + "signature": "3tW2EBVgxaZTE6nixVd9QEQf1vUxqPmQaNAMdCHc7zHik5KEosdkwScGYt4VhiHDTB6BCxTUzmqu3P7oP93tRWfj" + } diff --git a/docs/server/source/dev-and-test/index.rst b/docs/server/source/dev-and-test/index.rst index bcde6e87..b9ddaf58 100644 --- a/docs/server/source/dev-and-test/index.rst +++ b/docs/server/source/dev-and-test/index.rst @@ -7,7 +7,7 @@ This section outlines some ways that you could set up a minimal BigchainDB node :maxdepth: 1 Using a Local Dev Machine - Using a Local Dev Machine and Docker + Using a Local Dev Machine and Docker <../appendices/run-with-docker> Using Vagrant <../appendices/run-with-vagrant> Using Ansible <../appendices/run-with-ansible> - running-all-tests + running-all-tests \ No newline at end of file diff --git a/docs/server/source/dev-and-test/setup-bdb-docker.md b/docs/server/source/dev-and-test/setup-bdb-docker.md deleted file mode 100644 index a0a50dfd..00000000 --- a/docs/server/source/dev-and-test/setup-bdb-docker.md +++ /dev/null @@ -1,112 +0,0 @@ -# Set Up BigchainDB Node Using Docker - -You need to have recent versions of [Docker](https://docs.docker.com/engine/installation/) -and (Docker) [Compose](https://docs.docker.com/compose/install/). - -Build the images: - -```bash -docker-compose build -``` -## Docker with MongoDB - -Start MongoDB: - -```bash -docker-compose up -d mdb -``` - -MongoDB should now be up and running. You can check the port binding for the -MongoDB driver port using: -```bash -$ docker-compose port mdb 27017 -``` - -Start a BigchainDB node: - -```bash -docker-compose up -d bdb -``` - -You can monitor the logs: - -```bash -docker-compose logs -f bdb -``` - -If you wish to run the tests: - -```bash -docker-compose run --rm bdb py.test -v --database-backend=mongodb -``` -## Docker with RethinkDB - -**Note**: If you're upgrading BigchainDB and have previously already built the images, you may need -to rebuild them after the upgrade to install any new dependencies. - -Start RethinkDB: - -```bash -docker-compose -f docker-compose.rdb.yml up -d rdb -``` - -The RethinkDB web interface should be accessible at http://localhost:58080/. -Depending on which platform, and/or how you are running docker, you may need -to change `localhost` for the `ip` of the machine that is running docker. As a -dummy example, if the `ip` of that machine was `0.0.0.0`, you would access the -web interface at: http://0.0.0.0:58080/. - -Start a BigchainDB node: - -```bash -docker-compose -f docker-compose.rdb.yml up -d bdb-rdb -``` - -You can monitor the logs: - -```bash -docker-compose -f docker-compose.rdb.yml logs -f bdb-rdb -``` - -If you wish to run the tests: - -```bash -docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb pytest -v -n auto -``` - -## Accessing the HTTP API - -You can do quick check to make sure that the BigchainDB server API is operational: - -```bash -curl $(docker-compose port bdb 9984) -``` - -The result should be a JSON object (inside braces like { }) -containing the name of the software ("BigchainDB"), -the version of BigchainDB, the node's public key, and other information. - -How does the above curl command work? Inside the Docker container, BigchainDB -exposes the HTTP API on port `9984`. First we get the public port where that -port is bound: - -```bash -docker-compose port bdb 9984 -``` - -The port binding will change whenever you stop/restart the `bdb` service. You -should get an output similar to: - -```bash -0.0.0.0:32772 -``` - -but with a port different from `32772`. - - -Knowing the public port we can now perform a simple `GET` operation against the -root: - -```bash -curl 0.0.0.0:32772 -``` diff --git a/docs/server/source/glossary.rst b/docs/server/source/glossary.rst new file mode 100644 index 00000000..8fc03ac2 --- /dev/null +++ b/docs/server/source/glossary.rst @@ -0,0 +1,19 @@ +Glossary +======== + +.. glossary:: + :sorted: + + associative array + A collection of key/value (or name/value) pairs + such that each possible key appears at most once + in the collection. + In JavaScript (and JSON), all objects behave as associative arrays + with string-valued keys. + In Python and .NET, associative arrays are called *dictionaries*. + In Java and Go, they are called *maps*. + In Ruby, they are called *hashes*. + See also: Wikipedia's articles for + `Associative array `_ + and + `Comparison of programming languages (associative array) `_ diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index ec6efef0..65bd8774 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -16,7 +16,6 @@ BigchainDB Server Documentation events/index drivers-clients/index data-models/index - schema/transaction - schema/vote release-notes + glossary appendices/index diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index 371ecc36..492d15c6 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -424,13 +424,14 @@ LRS means locally-redundant storage: three replicas in the same data center. Premium storage is higher-cost and higher-performance. It uses solid state drives (SSD). -At the time of writing, -when we created a storage account with SKU ``Premium_LRS`` -and tried to use that, -the PersistentVolumeClaim would get stuck in a "Pending" state. +You can create a `storage account `_ +for Premium storage and associate it with your Azure resource group. For future reference, the command to create a storage account is `az storage account create `_. +.. Note:: + Please refer to `Azure documentation `_ + for the list of VMs that are supported by Premium Storage. The Kubernetes template for configuration of Storage Class is located in the file ``mongodb/mongo-sc.yaml``. @@ -438,6 +439,10 @@ file ``mongodb/mongo-sc.yaml``. You may have to update the ``parameters.location`` field in the file to specify the location you are using in Azure. +If you want to use a custom storage account with the Storage Class, you +can also update `parameters.storageAccount` and provide the Azure storage +account name. + Create the required storage classes using: .. code:: bash @@ -447,15 +452,6 @@ Create the required storage classes using: You can check if it worked using ``kubectl get storageclasses``. -**Azure.** Note that there is no line of the form -``storageAccount: `` -under ``parameters:``. When we included one -and then created a PersistentVolumeClaim based on it, -the PersistentVolumeClaim would get stuck -in a "Pending" state. -Kubernetes just looks for a storageAccount -with the specified skuName and location. - Step 11: Create Kubernetes Persistent Volume Claims --------------------------------------------------- diff --git a/docs/server/source/production-deployment-template/tectonic-azure.rst b/docs/server/source/production-deployment-template/tectonic-azure.rst index c59dc241..3803751e 100644 --- a/docs/server/source/production-deployment-template/tectonic-azure.rst +++ b/docs/server/source/production-deployment-template/tectonic-azure.rst @@ -47,7 +47,9 @@ when following the steps above: ``tectonic-cluster-CLUSTER``. #. Set the ``tectonic_base_domain`` to ``""`` if you want to use Azure managed - DNS. You will be assigned a ``cloudapp.azure.com`` sub-domain by default. + DNS. You will be assigned a ``cloudapp.azure.com`` sub-domain by default and + you can skip the ``Configuring Azure DNS`` section from the Tectonic installation + guide. #. Set the ``tectonic_cl_channel`` to ``"stable"`` unless you want to experiment or test with the latest release. @@ -76,6 +78,14 @@ when following the steps above: #. Set the ``tectonic_azure_ssh_key`` to the path of the public key created in the previous step. +#. We recommend setting up or using a CA(Certificate Authority) to generate Tectonic + Console's server certificate(s) and adding it to your trusted authorities on the client side, + accessing the Tectonic Console i.e. Browser. If you already have a CA(self-signed or otherwise), + Set the ``tectonic_ca_cert`` and ``tectonic_ca_key`` configurations with the content + of PEM-encoded certificate and key files, respectively. For more information about, how to set + up a self-signed CA, Please refer to + :doc:`How to Set up self-signed CA `. + #. Note that the ``tectonic_azure_client_secret`` is the same as the ``ARM_CLIENT_SECRET``. @@ -85,6 +95,10 @@ when following the steps above: ``test-cluster`` and specified the datacenter as ``westeurope``, the Tectonic console will be available at ``test-cluster.westeurope.cloudapp.azure.com``. +#. Note that, if you do not specify ``tectonic_ca_cert``, a CA certificate will + be generated automatically and you will encounter the untrusted certificate + message on your client(Browser), when accessing the Tectonic Console. + Step 4: Configure kubectl ------------------------- diff --git a/docs/server/source/production-deployment-template/template-kubernetes-azure.rst b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst index a916012f..7312ba36 100644 --- a/docs/server/source/production-deployment-template/template-kubernetes-azure.rst +++ b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst @@ -105,6 +105,21 @@ Finally, you can deploy an ACS using something like: --orchestrator-type kubernetes \ --debug --output json +.. Note:: + Please refer to `Azure documentation `_ + for a comprehensive list of options available for `az acs create`. + Please tune the following parameters as per your requirement: + + * Master count. + + * Agent count. + + * Agent VM size. + + * **Optional**: Master storage profile. + + * **Optional**: Agent storage profile. + There are more options. For help understanding all the options, use the built-in help: diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 63ab8643..e066a046 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -1,6 +1,6 @@ # Quickstart -This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. If you're not using Linux, then you might try [running BigchainDB with Docker](appendices/run-with-docker.html). +This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. You can also try, [running BigchainDB with Docker](appendices/run-with-docker.html). A. Install MongoDB as the database backend. (There are other options but you can ignore them for now.) diff --git a/k8s/mongodb/mongo-sc.yaml b/k8s/mongodb/mongo-sc.yaml index 2f291ffe..2b155d54 100644 --- a/k8s/mongodb/mongo-sc.yaml +++ b/k8s/mongodb/mongo-sc.yaml @@ -7,8 +7,12 @@ metadata: name: slow-db provisioner: kubernetes.io/azure-disk parameters: - skuName: Standard_LRS + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] location: westeurope + # If you have created a different storage account e.g. for Premium Storage + #storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed --- ###################################################################### # This YAML section desribes a StorageClass for the mongodb configDB # @@ -19,5 +23,9 @@ metadata: name: slow-configdb provisioner: kubernetes.io/azure-disk parameters: - skuName: Standard_LRS + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] location: westeurope + # If you have created a different storage account e.g. for Premium Storage + #storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed diff --git a/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash b/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash index 69bfd56f..e84e2563 100755 --- a/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash +++ b/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/nginx-https-web-proxy:0.10 . +docker build -t bigchaindb/nginx-https-web-proxy:0.12 . -docker push bigchaindb/nginx-https-web-proxy:0.10 +docker push bigchaindb/nginx-https-web-proxy:0.12 diff --git a/k8s/nginx-https-web-proxy/container/nginx.conf.template b/k8s/nginx-https-web-proxy/container/nginx.conf.template index 421d7fc0..c379dc8e 100644 --- a/k8s/nginx-https-web-proxy/container/nginx.conf.template +++ b/k8s/nginx-https-web-proxy/container/nginx.conf.template @@ -90,12 +90,6 @@ http { end } - # check if the request originated from the required web page - # use referer header. - if ($http_referer !~ "PROXY_EXPECTED_REFERER_HEADER" ) { - return 403 'Unknown referer'; - } - # check if the request has the expected origin header if ($http_origin !~ "PROXY_EXPECTED_ORIGIN_HEADER" ) { return 403 'Unknown origin'; @@ -108,9 +102,16 @@ http { add_header 'Access-Control-Max-Age' 43200; add_header 'Content-Type' 'text/plain charset=UTF-8'; add_header 'Content-Length' 0; + add_header 'Referrer-Policy' "PROXY_REFERRER_POLICY"; return 204; } + # check if the request originated from the required web page + # use referer header. + if ($http_referer !~ "PROXY_EXPECTED_REFERER_HEADER" ) { + return 403 'Unknown referer'; + } + # No auth for GETs, forward directly to BDB. if ($request_method = GET) { proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; diff --git a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml index b9dbc541..0a255d89 100644 --- a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml +++ b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml @@ -49,6 +49,11 @@ data: # are available to external clients. proxy-frontend-port: "4443" + # proxy-referrer-policy defines the expected behaviour from + # browser while setting the referer header in the HTTP requests to the + # proxy service. + proxy-referrer-policy: "origin-when-cross-origin" + # expected-http-referer is the expected regex expression of the Referer # header in the HTTP requests to the proxy. # The default below accepts the referrer value to be *.bigchaindb.com diff --git a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml index 6ada9347..e92aba7d 100644 --- a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml +++ b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml @@ -25,6 +25,11 @@ spec: configMapKeyRef: name: proxy-vars key: proxy-frontend-port + - name: PROXY_REFERRER_POLICY + valueFrom: + configMapKeyRef: + name: proxy-vars + key: proxy-referrer-policy - name: PROXY_EXPECTED_REFERER_HEADER valueFrom: configMapKeyRef: diff --git a/pkg/Vagrantfile b/pkg/Vagrantfile index b8720016..4031df07 100644 --- a/pkg/Vagrantfile +++ b/pkg/Vagrantfile @@ -9,10 +9,12 @@ Vagrant.require_version '>= 1.6.0' VAGRANTFILE_API_VERSION = '2' # Configuration files -CONFIGURATION_FILE = 'config/bdb-config.yaml' +CONFIGURATION_FILE = 'configuration/vars/bdb-config.yml' +HOSTS_FILE = 'configuration/hosts/all' +HOST_VARS_PATH = 'configuration/host_vars' # Validate if all the required plugins are present -required_plugins = ["vagrant-cachier"] +required_plugins = ["vagrant-cachier", "vagrant-vbguest", "vagrant-hosts"] required_plugins.each do |plugin| if not Vagrant.has_plugin?(plugin) raise "Required vagrant plugin #{plugin} not found. Please run `vagrant plugin install #{plugin}`" @@ -21,15 +23,28 @@ end # Read configuration file(s) instances_config = YAML.load_file(File.join(File.dirname(__FILE__), CONFIGURATION_FILE)) - -#TODO: (muawiakh) Add support for Docker, AWS, Azure +hosts_config = File.open(HOSTS_FILE, 'w+') +# TODO: (muawiakh) Add support for Docker, AWS, Azure Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - instances_config.each do |instance| + instances_config["bdb_hosts"].each do |instance| + # Workaround till canonical fixes https://bugs.launchpad.net/cloud-images/+bug/1569237 + # using -u ubuntu as remote user, conventionally vagrant boxes use `vagrant` user + if instance["box"]["name"] == "ubuntu/xenial64" + hosts_config.puts("#{instance["name"]} ansible_user=ubuntu") + else + hosts_config.puts("#{instance["name"]} ansible_user=vagrant") + end config.vm.define instance['name'] do |bdb| # Workaround until vagrant cachier plugin supports dnf if !(instance["box"]["name"].include? "fedora") if Vagrant.has_plugin?("vagrant-cachier") - config.cache.scope = :box + bdb.cache.scope = :box + end + elsif instance["box"]["name"] == "ubuntu/xenial64" + if Vagrant.has_plugin?("vagrant-vbguest") + bdb.vbguest.auto_update = false + bdb.vbguest.no_install = true + bdb.vbguest.no_remote = true end end bdb.vm.hostname = instance["name"] @@ -40,14 +55,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| else raise "Invalid network type: Please specify one of the following: [private_network, public_network]" end + bdb.vm.provision :hosts, :sync_hosts => true bdb.vm.box = instance["box"]["name"] bdb.vm.synced_folder ".", "/bigchaindb" - bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instance["upstart"]}" - if instance["setup_type"] == "quickstart" - bdb.vm.provision :shell, inline: "PYTHONBUFFERED=1 ansible-playbook \ - /bigchaindb/ansible/quickstart.yml --extra-vars \"with_docker=#{instance["deploy_docker"]}\" -c local" - end - + File.open("#{HOST_VARS_PATH}/#{instance["name"]}", "w+") {|f| \ + f.write("ansible_ssh_private_key_file: /bigchaindb/.vagrant/machines/#{instance["name"]}/virtualbox/private_key") } + bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" bdb.vm.provider 'vmware_fusion' do |vmwf, override| vmwf.vmx['memsize'] = instance["ram"] vmwf.vmx['numvcpus'] = instance['vcpus'] @@ -59,4 +72,32 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end end end + hosts_config.close + config.vm.define "config-node" do |bdb| + bdb.vm.box = "ubuntu/xenial64" + bdb.vm.hostname = "config-node" + bdb.vm.provision :hosts, :sync_hosts => true + bdb.vm.synced_folder ".", "/bigchaindb" + bdb.vm.network "private_network", ip: "192.168.100.200" + bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" + bdb.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /bigchaindb/configuration/bdb-deploy.yml \ + -i /bigchaindb/configuration/hosts/all" + bdb.vm.provider "virtualbox" do |vb| + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + vb.memory = 2048 + vb.cpus = 2 + end + bdb.vm.provider 'vmware_fusion' do |vmwf| + vmwf.vmx['memsize'] = 2048 + vmwf.vmx['numvcpus'] = 2 + end + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + config.vbguest.no_install = true + config.vbguest.no_remote = true + end + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + end end diff --git a/pkg/ansible/quickstart.yml b/pkg/ansible/quickstart.yml deleted file mode 100644 index aa0f8c49..00000000 --- a/pkg/ansible/quickstart.yml +++ /dev/null @@ -1,10 +0,0 @@ -- hosts: localhost - remote_user: vagrant - vars: - with_docker: "{{ deploy_docker | default(false) }}" - roles: - - { role: docker, when: with_docker|bool } - - { role: docker-compose, when: with_docker|bool } - - mongodb - - bigchaindb - - bigchaindb-driver \ No newline at end of file diff --git a/pkg/ansible/roles/bigchaindb/tasks/main.yml b/pkg/ansible/roles/bigchaindb/tasks/main.yml deleted file mode 100644 index 532c4ae6..00000000 --- a/pkg/ansible/roles/bigchaindb/tasks/main.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- include: with_docker.yml - when: with_docker|bool - tags: [bigchaindb] - -- include: debian.yml - when: not with_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") - -- include: centos.yml - when: not with_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") - -- include: fedora.yml - when: not with_docker|bool and (distribution_name == "fedora") - -- include: common.yml - when: not with_docker|bool \ No newline at end of file diff --git a/pkg/ansible/roles/bigchaindb/tasks/with_docker.yml b/pkg/ansible/roles/bigchaindb/tasks/with_docker.yml deleted file mode 100644 index bf832711..00000000 --- a/pkg/ansible/roles/bigchaindb/tasks/with_docker.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Configuring BigchainDB Docker - docker_container: - name: "{{ bigchaindb_docker_name }}" - image: "{{ bigchaindb_image_name }}" - volumes: "{{ bigchaindb_docker_volumes }}" - pull: false - env: - BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" - BIGCHAINDB_DATABASE_HOST: "{{ bigchaindb_database_host }}" - entrypoint: "bigchaindb -y configure mongodb" - register: result - tags: [bigchaindb] - -- name: Start BigchainDB Docker - docker_container: - name: "{{ bigchaindb_docker_name }}" - image: "{{ bigchaindb_image_name }}" - published_ports: "{{ bigchaindb_docker_published_ports }}" - restart_policy: always - volumes: "{{ bigchaindb_docker_volumes }}" - state: started - pull: false - when: result|succeeded - tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/ansible/roles/mongodb/tasks/common.yml b/pkg/ansible/roles/mongodb/tasks/common.yml deleted file mode 100644 index 41c6de1d..00000000 --- a/pkg/ansible/roles/mongodb/tasks/common.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: MongoDB Process Check - shell: pgrep mongod | wc -l - register: command_result - tags: [mongodb] - -- name: Run MongoDB - shell: "mongod --replSet=bigchain-rs --logpath {{ mongodb_log_path }}/mongod.log &" - when: command_result.stdout| int != 1 - tags: [mongodb] \ No newline at end of file diff --git a/pkg/ansible/roles/mongodb/tasks/main.yml b/pkg/ansible/roles/mongodb/tasks/main.yml deleted file mode 100644 index 451b81b9..00000000 --- a/pkg/ansible/roles/mongodb/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: Creating directories - file: - path: "{{ item }}" - state: directory - mode: 0700 - with_items: "{{ directories }}" - tags: [mongodb] - -- include: with_docker.yml - when: with_docker|bool - -- name: Verify logfiles exist | Debian - file: - path: "{{ mongodb_log_path }}/mongod.log" - state: touch - mode: 0755 - when: not with_docker|bool - tags: [mongodb] - -- include: debian.yml - when: not with_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") - -- include: centos.yml - when: not with_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") - -- include: fedora.yml - when: not with_docker|bool and (distribution_name == "fedora") - -- include: common.yml - when: not with_docker|bool \ No newline at end of file diff --git a/pkg/ansible/roles/mongodb/tasks/with_docker.yml b/pkg/ansible/roles/mongodb/tasks/with_docker.yml deleted file mode 100644 index ff3a24a5..00000000 --- a/pkg/ansible/roles/mongodb/tasks/with_docker.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Check Docker Service - systemd: - name: docker - enabled: yes - state: started - tags: [docker] - -- name: Running MongoDB Docker - docker_container: - name: "{{ mongodb_docker_name }}" - image: "{{ mongodb_docker_image }}" - detach: True - published_ports: "{{ mongodb_docker_published_ports }}" - restart_policy: always - volumes: "{{ mongodb_docker_volumes }}" - state: started - pull: false - entrypoint: /entrypoint.sh --replSet=bigchain-rs - tags: [mongodb] \ No newline at end of file diff --git a/pkg/config/bdb-config.yaml b/pkg/config/bdb-config.yaml deleted file mode 100644 index 86acac56..00000000 --- a/pkg/config/bdb-config.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: "bdb-node-01" # Instance name - box: - name: "ubuntu/xenial64" # Box name - ram: "2048" - vcpus: "2" - setup_type: "quickstart" # Currently, only quickstart is supported. - deploy_docker: true # [true, false] - network: - ip: "10.20.30.50" - type: "private_network" - # Active network interface on host, Only required for public network e.g "en0: Wi-Fi (AirPort)" - bridge: "" - upstart: "/bigchaindb/scripts/bootstrap.sh" # Path to upstart script diff --git a/pkg/configuration/bdb-deploy.yml b/pkg/configuration/bdb-deploy.yml new file mode 100644 index 00000000..8dd8c7ea --- /dev/null +++ b/pkg/configuration/bdb-deploy.yml @@ -0,0 +1,12 @@ +- import_playbook: pre_req.yml + +- hosts: all + vars_files: + - vars/bdb-config.yml + serial: 1 + roles: + - bigchaindb + - bigchaindb-driver + +- import_playbook: multi_node.yml + when: (bdb_hosts|length > 1) or docker_cluster_size|int > 1 diff --git a/pkg/configuration/group_vars/all b/pkg/configuration/group_vars/all new file mode 100644 index 00000000..f530443e --- /dev/null +++ b/pkg/configuration/group_vars/all @@ -0,0 +1,5 @@ +--- +ansible_connection: ssh +ansible_ssh_port: 22 +ansible_become: yes +ansible_ssh_common_args: '-o StrictHostKeyChecking=no' \ No newline at end of file diff --git a/pkg/configuration/host_vars/bdb-node-01 b/pkg/configuration/host_vars/bdb-node-01 new file mode 100644 index 00000000..fef41e4a --- /dev/null +++ b/pkg/configuration/host_vars/bdb-node-01 @@ -0,0 +1,5 @@ +# Place holder file for users, running Ansible playbooks manually. Otherwise Vagrant +# populates this dynamically. + +# Only needed for logging into remote hosts and adding host specific variables e.g. +#ansible_ssh_private_key_file: "/path/to/private/key" diff --git a/pkg/configuration/hosts/all b/pkg/configuration/hosts/all new file mode 100644 index 00000000..f84cc374 --- /dev/null +++ b/pkg/configuration/hosts/all @@ -0,0 +1,8 @@ +# Place holder file for users, running Ansible playbooks manually. Otherwise Vagrant +# populates this dynamically. + +# For local host +# ansible_connection=local + +# For remote host(s) +# ansible_ssh_user= ansible_sudo_pass= diff --git a/pkg/configuration/multi_node.yml b/pkg/configuration/multi_node.yml new file mode 100644 index 00000000..aa293ad9 --- /dev/null +++ b/pkg/configuration/multi_node.yml @@ -0,0 +1,5 @@ +- hosts: all + vars_files: + - vars/bdb-config.yml + roles: + - key-exchange \ No newline at end of file diff --git a/pkg/configuration/pre_req.yml b/pkg/configuration/pre_req.yml new file mode 100644 index 00000000..b7e54581 --- /dev/null +++ b/pkg/configuration/pre_req.yml @@ -0,0 +1,8 @@ +- hosts: all + vars_files: + - vars/bdb-config.yml + serial: 1 + roles: + - { role: docker, when: deploy_docker|bool } + - { role: docker-compose, when: deploy_docker|bool } + - mongodb \ No newline at end of file diff --git a/pkg/ansible/roles/bigchaindb-driver/defaults/main.yml b/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml similarity index 62% rename from pkg/ansible/roles/bigchaindb-driver/defaults/main.yml rename to pkg/configuration/roles/bigchaindb-driver/defaults/main.yml index 63485cce..28c99323 100644 --- a/pkg/ansible/roles/bigchaindb-driver/defaults/main.yml +++ b/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml @@ -23,4 +23,9 @@ dependencies_dnf: - python3-pip python_pip_upgrade: true -python_setuptools_upgrade: true \ No newline at end of file +python_setuptools_upgrade: true + +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" \ No newline at end of file diff --git a/pkg/ansible/roles/bigchaindb-driver/tasks/centos.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb-driver/tasks/centos.yml rename to pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml diff --git a/pkg/ansible/roles/bigchaindb-driver/tasks/common.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb-driver/tasks/common.yml rename to pkg/configuration/roles/bigchaindb-driver/tasks/common.yml diff --git a/pkg/ansible/roles/bigchaindb-driver/tasks/debian.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb-driver/tasks/debian.yml rename to pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml diff --git a/pkg/ansible/roles/bigchaindb-driver/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb-driver/tasks/fedora.yml rename to pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml diff --git a/pkg/ansible/roles/bigchaindb-driver/tasks/main.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml similarity index 65% rename from pkg/ansible/roles/bigchaindb-driver/tasks/main.yml rename to pkg/configuration/roles/bigchaindb-driver/tasks/main.yml index f743ff1c..f70ef304 100644 --- a/pkg/ansible/roles/bigchaindb-driver/tasks/main.yml +++ b/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml @@ -1,12 +1,12 @@ --- -- include: debian.yml +- import_tasks: debian.yml when: distribution_name == "debian" or distribution_name == "ubuntu" -- include: centos.yml +- import_tasks: centos.yml when: distribution_name == "centos" or distribution_name == "red hat enterprise linux" -- include: fedora.yml +- import_tasks: fedora.yml when: distribution_name == "fedora" -- include: common.yml \ No newline at end of file +- import_tasks: common.yml \ No newline at end of file diff --git a/pkg/ansible/roles/bigchaindb/defaults/main.yml b/pkg/configuration/roles/bigchaindb/defaults/main.yml similarity index 59% rename from pkg/ansible/roles/bigchaindb/defaults/main.yml rename to pkg/configuration/roles/bigchaindb/defaults/main.yml index 83058813..594deb20 100644 --- a/pkg/ansible/roles/bigchaindb/defaults/main.yml +++ b/pkg/configuration/roles/bigchaindb/defaults/main.yml @@ -27,21 +27,25 @@ dependencies_dnf: python_pip_upgrade: true python_setuptools_upgrade: true +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" + directories: - /data -backend_db: mongodb #[rethinkdb, mongodb] +backend_db: mongodb #[mongodb] +bigchaindb_config_path: /data/.bigchaindb bigchaindb_server_bind: "0.0.0.0:9984" -bigchaindb_database_host: "172.17.0.1" bigchaindb_log_file: "{{ ansible_env.HOME }}/bigchaindb.log" # Docker configuration -backend_db_image: "mongo:3.4.1" -backend_db_name: "mongodb" bigchaindb_image_name: "bigchaindb/bigchaindb" bigchaindb_docker_name: "bigchaindb" -bigchaindb_docker_published_ports: - - 59984:9984 -bigchaindb_docker_volumes: - - "{{ ansible_env.HOME }}/bigchaindb_docker:/data" +bigchaindb_default_port: 9984 +bigchandb_host_port: 59984 +bigchaindb_host_mount_dir: "{{ ansible_env.HOME }}/bigchaindb_docker" +# Default IP of docker0 bridge +bigchaindb_default_host: "172.17.0.1" diff --git a/pkg/ansible/roles/bigchaindb/tasks/centos.yml b/pkg/configuration/roles/bigchaindb/tasks/centos.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb/tasks/centos.yml rename to pkg/configuration/roles/bigchaindb/tasks/centos.yml diff --git a/pkg/ansible/roles/bigchaindb/tasks/common.yml b/pkg/configuration/roles/bigchaindb/tasks/common.yml similarity index 53% rename from pkg/ansible/roles/bigchaindb/tasks/common.yml rename to pkg/configuration/roles/bigchaindb/tasks/common.yml index c88882be..d29cc3ef 100644 --- a/pkg/ansible/roles/bigchaindb/tasks/common.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/common.yml @@ -13,10 +13,18 @@ shell: "pip3 install bigchaindb" tags: [bigchaindb] +- name: Check if BigchainDB node is already configured + stat: + path: "{{ bigchaindb_config_path }}" + register: stat_result + - name: Configure BigchainDB shell: "bigchaindb -y configure {{ backend_db }}" environment: BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" + BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path }}" + BIGCHAINDB_DATABASE_HOST: "{{ ansible_hostname }}" + when: stat_result.stat.exists == False tags: [bigchaindb] - name: MongoDB Process Check @@ -30,7 +38,22 @@ tags: [bigchaindb] - name: Start BigchainDB - become: yes shell: "bigchaindb start > {{ bigchaindb_log_file }} 2>&1 &" + environment: + BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path }}" when: mdb_pchk.stdout| int >= 1 and bdb_pchk.stdout| int == 0 - tags: [bigchaindb] \ No newline at end of file + async: 10 + poll: 0 + tags: [bigchaindb] + +- name: Get BigchainDB node public key + shell: "cat {{ bigchaindb_config_path }}" + register: bdb_node_config + tags: [bigchaindb] + +- name: Set Facts BigchainDB + set_fact: + pub_key="{{ ( bdb_node_config.stdout|from_json).keypair.public }}" + hostname="{{ ansible_hostname }}" + bdb_config="{{ bigchaindb_config_path }}" + tags: [bigchaindb] diff --git a/pkg/ansible/roles/bigchaindb/tasks/debian.yml b/pkg/configuration/roles/bigchaindb/tasks/debian.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb/tasks/debian.yml rename to pkg/configuration/roles/bigchaindb/tasks/debian.yml diff --git a/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml new file mode 100644 index 00000000..98fd90d5 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml @@ -0,0 +1,48 @@ +--- +- name: Check if BigchainDB Dockers are already configured + stat: + path: "{{ bigchaindb_host_mount_dir }}{{ item|string }}/.bigchaindb" + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + register: stat_result + tags: [bigchaindb] + +- name: Configuring BigchainDB Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + hostname: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}" + volumes: + - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" + env: + BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" + BIGCHAINDB_DATABASE_HOST: "{{ hostvars[ansible_hostname]['mongodb' + item|string] }}" + entrypoint: "bigchaindb -y configure mongodb" + when: stat_result.results[item|int].stat.exists == False + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] + +- name: Start BigchainDB Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}" + detach: true + published_ports: + - "{{ bigchandb_host_port|int + item|int }}:{{ bigchaindb_default_port }}" + restart_policy: always + volumes: + - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" + state: started + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] + +- name: Get BigchainDB node public key + shell: "cat {{ bigchaindb_host_mount_dir + item|string }}/.bigchaindb" + register: bdb_node_config + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] + +- name: Set facts for BigchainDB containers + set_fact: + pub_key_{{ bigchaindb_docker_name }}{{ item }}="{{ (bdb_node_config.results[item|int].stdout|from_json).keypair.public }}" + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] diff --git a/pkg/ansible/roles/bigchaindb/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml similarity index 100% rename from pkg/ansible/roles/bigchaindb/tasks/fedora.yml rename to pkg/configuration/roles/bigchaindb/tasks/fedora.yml diff --git a/pkg/configuration/roles/bigchaindb/tasks/main.yml b/pkg/configuration/roles/bigchaindb/tasks/main.yml new file mode 100644 index 00000000..3b1de267 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- import_tasks: deploy_docker.yml + when: deploy_docker|bool + tags: [bigchaindb] + +- import_tasks: debian.yml + when: not deploy_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [bigchaindb] + +- import_tasks: centos.yml + when: not deploy_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [bigchaindb] + +- import_tasks: fedora.yml + when: not deploy_docker|bool and (distribution_name == "fedora") + tags: [bigchaindb] + +- import_tasks: common.yml + when: not deploy_docker|bool + tags: [bigchaindb] diff --git a/pkg/ansible/roles/docker-compose/defaults/main.yml b/pkg/configuration/roles/docker-compose/defaults/main.yml similarity index 100% rename from pkg/ansible/roles/docker-compose/defaults/main.yml rename to pkg/configuration/roles/docker-compose/defaults/main.yml diff --git a/pkg/ansible/roles/docker-compose/tasks/main.yml b/pkg/configuration/roles/docker-compose/tasks/main.yml similarity index 100% rename from pkg/ansible/roles/docker-compose/tasks/main.yml rename to pkg/configuration/roles/docker-compose/tasks/main.yml diff --git a/pkg/ansible/roles/docker/defaults/main.yml b/pkg/configuration/roles/docker/defaults/main.yml similarity index 100% rename from pkg/ansible/roles/docker/defaults/main.yml rename to pkg/configuration/roles/docker/defaults/main.yml diff --git a/pkg/ansible/roles/docker/tasks/centos.yml b/pkg/configuration/roles/docker/tasks/centos.yml similarity index 100% rename from pkg/ansible/roles/docker/tasks/centos.yml rename to pkg/configuration/roles/docker/tasks/centos.yml diff --git a/pkg/ansible/roles/docker/tasks/debian.yml b/pkg/configuration/roles/docker/tasks/debian.yml similarity index 100% rename from pkg/ansible/roles/docker/tasks/debian.yml rename to pkg/configuration/roles/docker/tasks/debian.yml diff --git a/pkg/ansible/roles/docker/tasks/fedora.yml b/pkg/configuration/roles/docker/tasks/fedora.yml similarity index 100% rename from pkg/ansible/roles/docker/tasks/fedora.yml rename to pkg/configuration/roles/docker/tasks/fedora.yml diff --git a/pkg/ansible/roles/docker/tasks/main.yml b/pkg/configuration/roles/docker/tasks/main.yml similarity index 87% rename from pkg/ansible/roles/docker/tasks/main.yml rename to pkg/configuration/roles/docker/tasks/main.yml index 66d36489..5676e153 100644 --- a/pkg/ansible/roles/docker/tasks/main.yml +++ b/pkg/configuration/roles/docker/tasks/main.yml @@ -1,11 +1,11 @@ --- -- include: debian.yml +- import_tasks: debian.yml when: distribution_name == "debian" or distribution_name == "ubuntu" -- include: centos.yml +- import_tasks: centos.yml when: distribution_name == "centos" or distribution_name == "red hat enterprise linux" -- include: fedora.yml +- import_tasks: fedora.yml when: distribution_name == "fedora" - name: Create Docker group diff --git a/pkg/configuration/roles/key-exchange/defaults/main.yml b/pkg/configuration/roles/key-exchange/defaults/main.yml new file mode 100644 index 00000000..6bfe9035 --- /dev/null +++ b/pkg/configuration/roles/key-exchange/defaults/main.yml @@ -0,0 +1,12 @@ +keyring_script_host: /tmp/keyring.py +bigchaindb_log_file_host: "{{ ansible_env.HOME }}/bigchaindb.log" +bigchaindb_config_path_host: /data/.bigchaindb + +# Docker configuration +keyring_script_docker: "{{ ansible_env.HOME }}/config/keyring.py" +bigchaindb_config_path_docker: "{{ ansible_env.HOME }}/bigchaindb_docker" +bigchaindb_docker_name: bigchaindb +bigchaindb_default_port: 9984 +bigchandb_host_port: 59984 +bigchaindb_host_mount_dir: "{{ ansible_env.HOME }}/bigchaindb_docker" +bigchaindb_image_name: "bigchaindb/bigchaindb" diff --git a/pkg/configuration/roles/key-exchange/tasks/main.yml b/pkg/configuration/roles/key-exchange/tasks/main.yml new file mode 100644 index 00000000..031bf16b --- /dev/null +++ b/pkg/configuration/roles/key-exchange/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- include_tasks: pub_key_exchange_host.yml + when: not deploy_docker|bool + tags: [bigchaindb] + +- include_tasks: pub_key_exchange_docker.yml + when: deploy_docker|bool + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml new file mode 100644 index 00000000..eaf0eb14 --- /dev/null +++ b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml @@ -0,0 +1,29 @@ +--- +- name: Creating files for key exchange + template: src=exchange_keyring_docker.j2 dest="{{ keyring_script_docker }}" + tags: [keyex] + +- name: Setting permissions + file: + path: "{{ keyring_script_docker }}" + mode: "0777" + tags: [keyex] + +- name: Update Keyring Configuration + shell: "python {{ keyring_script_docker }}" + tags: [keyex] + +- name: Restart BigchainDB Docker after keyring update + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}" + detach: true + published_ports: + - "{{ bigchandb_host_port|int + item|int }}:{{ bigchaindb_default_port }}" + restart_policy: always + volumes: + - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" + state: started + restart: true + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [keyex] diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml new file mode 100644 index 00000000..f3d5c4d6 --- /dev/null +++ b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml @@ -0,0 +1,28 @@ +--- +- name: Creating files for key exchange + template: src=exchange_keyring_host.j2 dest="{{ keyring_script_host }}" + tags: [keyex] + +- name: Setting permissions + file: + path: "{{ keyring_script_host }}" + mode: "0777" + tags: [keyex] + +- name: Update Keyring Configuration + shell: "python {{ keyring_script_host }}" + tags: [keyex] + +- name: Stop BigchainDB + shell: pkill bigchaindb + register: bdb_stop + tags: [keyex] + +- name: Start BigchainDB + shell: "bigchaindb start > {{ bigchaindb_log_file_host }} 2>&1 &" + environment: + BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path_host }}" + async: 10 + poll: 0 + when: bdb_stop|succeeded + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 new file mode 100644 index 00000000..31aed4fb --- /dev/null +++ b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 @@ -0,0 +1,18 @@ +#!/usr/bin/python +import json +{% set keyring = {} %} +{% for docker in range(0, docker_cluster_size|int, 1) %} + {{- keyring.update({'pub_key_' + bigchaindb_docker_name + docker|string: hostvars[ansible_hostname]['pub_key_' + bigchaindb_docker_name + docker|string]}) -}} +{%- endfor -%} +{% for docker in range(0, docker_cluster_size|int, 1) %} +keyring = {{ keyring }} +keyring.pop('{{ 'pub_key_' + bigchaindb_docker_name + docker|string }}', None) +with open('{{ bigchaindb_config_path_docker + docker|string }}/.bigchaindb', 'r+') as f: + data = json.load(f) + del data['keyring'][:] + for key, value in keyring.iteritems(): + data['keyring'].append(value) + f.seek(0) + json.dump(data, f, indent=4) + f.truncate() +{% endfor %} \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 new file mode 100644 index 00000000..f02a208c --- /dev/null +++ b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 @@ -0,0 +1,21 @@ +{%- set keyring = [] -%} +{%- set bdb_config_path = {'path': ''} -%} +{%- for host in bdb_hosts -%} + {%- if host["name"] != ansible_hostname -%} + {{- keyring.append(hostvars[host["name"]]["pub_key"]) -}} + {%- else -%} + {%- if bdb_config_path.update({'path': hostvars[host["name"]]["bdb_config"]}) -%} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if keyring|length != 0 -%} +#!/usr/bin/python +import json +with open('{{ bdb_config_path['path'] }}', 'r+') as f: + data = json.load(f) + del data['keyring'][:] + data['keyring'] = {{ keyring }} + f.seek(0) + json.dump(data, f, indent=4) + f.truncate() +{%- endif -%} \ No newline at end of file diff --git a/pkg/ansible/roles/mongodb/defaults/main.yml b/pkg/configuration/roles/mongodb/defaults/main.yml similarity index 74% rename from pkg/ansible/roles/mongodb/defaults/main.yml rename to pkg/configuration/roles/mongodb/defaults/main.yml index 950b4a18..7088a61b 100644 --- a/pkg/ansible/roles/mongodb/defaults/main.yml +++ b/pkg/configuration/roles/mongodb/defaults/main.yml @@ -17,20 +17,23 @@ mongodb_dnf_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower } mongodb_storage_path: /data/db/main mongodb_log_path: /var/log/mongodb mongodb_config_path: /data/configdb -se_linux: "TODO: (muawiakh)" directories: - "{{ mongodb_storage_path }}" - "{{ mongodb_log_path }}" - "{{ mongodb_config_path }}" +mongodb_conf_file: /etc/mongod.conf +mongodb_conf_files: [ + { src: "mongod.conf", dest: "{{ mongodb_conf_file }}"} +] +mongodb_port: 27017 +mongodb_admin_user: "adminUser" +mongodb_admin_password: "superstrongpassword" + +replica_set_name: bigchain-rs + # Docker configuration -mongodb_default_port: 27017 -mongodb_docker_image: "mongo:3.4.4" +mongodb_docker_image: "mongo:3.4.9" mongodb_docker_name: "mongodb" -mongodb_docker_published_ports: - - 172.17.0.1:27017:27017 -mongodb_host_mount_dir_db: /tmp/mongodb_docker/db -mongodb_host_mount_dir_config: /tmp/mongodb_docker/configdb -mongodb_docker_volumes: - - "{{ mongodb_host_mount_dir_db }}:{{ mongodb_storage_path }}" - - "{{ mongodb_host_mount_dir_config }}:{{ mongodb_config_path }}" \ No newline at end of file +mongodb_host_mount_dir: "{{ ansible_env.HOME }}/mongodb_docker" +mongodb_host_config: "{{ ansible_env.HOME }}/config" diff --git a/pkg/configuration/roles/mongodb/files/mongod.conf b/pkg/configuration/roles/mongodb/files/mongod.conf new file mode 100644 index 00000000..ed961801 --- /dev/null +++ b/pkg/configuration/roles/mongodb/files/mongod.conf @@ -0,0 +1,101 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# where to write logging data. +systemLog: + verbosity: 0 + # traceAllExceptions: true + timeStampFormat: iso8601-utc + component: + accessControl: + verbosity: 0 + command: + verbosity: 0 + control: + verbosity: 0 + ftdc: + verbosity: 0 + geo: + verbosity: 0 + index: + verbosity: 0 + network: + verbosity: 0 + query: + verbosity: 0 + replication: + verbosity: 0 + sharding: + verbosity: 0 + storage: + verbosity: 0 + journal: + verbosity: 0 + write: + verbosity: 0 + +processManagement: + fork: false + pidFilePath: /tmp/mongod.pid + +net: + port: 27017 + bindIp: 0.0.0.0 + maxIncomingConnections: 8192 + wireObjectCheck: false + unixDomainSocket: + enabled: false + pathPrefix: /tmp + filePermissions: 0700 + http: + enabled: false + compression: + compressors: snappy +# ssl: +# mode: requireSSL +# PEMKeyFile: MONGODB_KEY_FILE_PATH +# PEMKeyPassword: MONGODB_KEY_FILE_PASSWORD +# CAFile: MONGODB_CA_FILE_PATH +# CRLFile: MONGODB_CRL_FILE_PATH + +# allowConnectionsWithoutCertificates: false +# allowInvalidHostnames: false +# weakCertificateValidation: false +# allowInvalidCertificates: false + +#security: +# authorization: enabled +# clusterAuthMode: x509 + +#setParameter: +# enableLocalhostAuthBypass: true +# #notablescan: 1 +# logUserIds: 1 +# authenticationMechanisms: MONGODB-X509,SCRAM-SHA-1 + +storage: + dbPath: /data/db/main + indexBuildRetry: true + journal: + enabled: true + commitIntervalMs: 100 + directoryPerDB: true + engine: wiredTiger + wiredTiger: + engineConfig: + journalCompressor: snappy +# configString: cache_size=STORAGE_ENGINE_CACHE_SIZE + collectionConfig: + blockCompressor: snappy + indexConfig: + prefixCompression: true # TODO false may affect performance? + +operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + +replication: + replSetName: bigchain-rs + enableMajorityReadConcern: true diff --git a/pkg/ansible/roles/mongodb/tasks/centos.yml b/pkg/configuration/roles/mongodb/tasks/centos.yml similarity index 76% rename from pkg/ansible/roles/mongodb/tasks/centos.yml rename to pkg/configuration/roles/mongodb/tasks/centos.yml index e3340100..62e8faa0 100644 --- a/pkg/ansible/roles/mongodb/tasks/centos.yml +++ b/pkg/configuration/roles/mongodb/tasks/centos.yml @@ -15,4 +15,11 @@ name: "{{ mongodb_package }}" state: present update_cache: yes + tags: [mongodb] + +- name: Install pip | CentOS + yum: + name: python-pip + state: present + update_cache: yes tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/common.yml b/pkg/configuration/roles/mongodb/tasks/common.yml new file mode 100644 index 00000000..40369191 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/common.yml @@ -0,0 +1,25 @@ +--- +- name: MongoDB config files are copied + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: "{{ mongodb_conf_files }}" + tags: [mongodb] + +- name: MongoDB Process Check + shell: pgrep mongod | wc -l + register: command_result + tags: [mongodb] + +- name: Install pymongo + pip: + name: pymongo + state: present + tags: [mongodb] + +- name: Run MongoDB + shell: "mongod --config {{ mongodb_conf_file }} 2>&1 &" + when: command_result.stdout| int != 1 + async: 5 + poll: 0 + tags: [mongodb] \ No newline at end of file diff --git a/pkg/ansible/roles/mongodb/tasks/debian.yml b/pkg/configuration/roles/mongodb/tasks/debian.yml similarity index 78% rename from pkg/ansible/roles/mongodb/tasks/debian.yml rename to pkg/configuration/roles/mongodb/tasks/debian.yml index c2b0349b..aac606fa 100644 --- a/pkg/ansible/roles/mongodb/tasks/debian.yml +++ b/pkg/configuration/roles/mongodb/tasks/debian.yml @@ -3,8 +3,6 @@ apt_key: keyserver: "{{ apt_keyserver }}" id: "{{ apt_key_fingerprint }}" - state: present - ignore_errors: true tags: [mongodb] - name: Add MongoDB repo and update cache | Debian @@ -16,6 +14,9 @@ - name: Install MongoDB | Debian apt: - name: "{{ mongodb_package }}" + name: "{{ item }}" state: present + with_items: + - "{{ mongodb_package }}" + - python-pip tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml new file mode 100644 index 00000000..ecadea49 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml @@ -0,0 +1,33 @@ +--- +- name: Check Docker Service + systemd: + name: docker + enabled: yes + state: started + tags: [mongodb] + +- name: Running MongoDB Docker + docker_container: + name: "{{ mongodb_docker_name }}{{ item }}" + hostname: "{{ mongodb_docker_name }}{{ item }}" + image: "{{ mongodb_docker_image }}" + detach: true + published_ports: + - "{{ (mongodb_port|int + item|int)|string }}:{{ mongodb_port }}" + restart_policy: always + volumes: + - "{{ mongodb_host_mount_dir }}{{ item|string }}/db:{{ mongodb_storage_path }}" + - "{{ mongodb_host_mount_dir }}{{ item|string }}/configdb:{{ mongodb_config_path }}" + - "{{ mongodb_host_config }}:/bdb_config" + state: started + keep_volumes: true + entrypoint: /entrypoint.sh --replSet=bigchain-rs + register: mongo_container_info + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [mongodb] + +- name: Set facts for MongoDB containers + set_fact: + mongodb{{ item }}={{ mongo_container_info.results[item|int].ansible_facts.docker_container.NetworkSettings.IPAddress }} + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [mongodb] diff --git a/pkg/ansible/roles/mongodb/tasks/fedora.yml b/pkg/configuration/roles/mongodb/tasks/fedora.yml similarity index 82% rename from pkg/ansible/roles/mongodb/tasks/fedora.yml rename to pkg/configuration/roles/mongodb/tasks/fedora.yml index fb83357f..c2f61110 100644 --- a/pkg/ansible/roles/mongodb/tasks/fedora.yml +++ b/pkg/configuration/roles/mongodb/tasks/fedora.yml @@ -12,6 +12,8 @@ - name: Install MongoDB | Fedora dnf: - name: "{{ mongodb_package }}" + name: "{{ item }}" state: present + with_items: + - "{{ mongodb_package }}" tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml new file mode 100644 index 00000000..ecbdc625 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml @@ -0,0 +1,6 @@ +--- +- import_tasks: initiate_repl_set_host.yml + when: (ansible_hostname == bdb_hosts[bdb_hosts|length-1]['name']) and not deploy_docker|bool + +- import_tasks: initiate_repl_set_docker.yml + when: deploy_docker|bool and docker_cluster_size|int > 1 diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml new file mode 100644 index 00000000..5cd341fa --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml @@ -0,0 +1,13 @@ +--- +- name: Creating files to initialize MongoDB Replica Set | Docker + template: src=replSet_init_docker.j2 dest="{{ mongodb_host_config }}/replSet_init.js" + tags: [mongodb] + +- name: Initializing Replica Set and Adding AdminUser | Docker + run_once: true + shell: + cmd: + "docker exec {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} bash -l -c + '/usr/bin/mongo --host {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} + --port {{ mongodb_port }} < /bdb_config/replSet_init.js'" + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml new file mode 100644 index 00000000..8d398bcb --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml @@ -0,0 +1,20 @@ +--- +- name: Creating files to initialize MongoDB Replica Set + template: src=replSet_init_host.j2 dest=/tmp/replSet_init.js + tags: [mongodb] + +- name: Initializing Replica Set + shell: "/usr/bin/mongo --host {{ ansible_hostname }} --port {{ mongodb_port }} < /tmp/replSet_init.js" + tags: [mongodb] + +- name: Adding AdminUser to MongoDB + run_once: true + mongodb_user: + database: admin + login_host: "{{ ansible_hostname }}" + login_port: "{{ mongodb_port }}" + name: "{{ mongodb_admin_user }}" + password: "{{ mongodb_admin_password }}" + roles: readWriteAnyDatabase,clusterManager + state: present + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/main.yml b/pkg/configuration/roles/mongodb/tasks/main.yml new file mode 100644 index 00000000..8daeffc6 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Creating directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + when: not deploy_docker|bool + tags: [mongodb] + +- import_tasks: deploy_docker.yml + when: deploy_docker|bool + tags: [mongodb] + +- import_tasks: debian.yml + when: not deploy_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [mongodb] + +- import_tasks: centos.yml + when: not deploy_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [mongodb] + +- import_tasks: fedora.yml + when: not deploy_docker|bool and (distribution_name == "fedora") + tags: [mongodb] + +- import_tasks: common.yml + when: not deploy_docker|bool + tags: [mongodb] + +- import_tasks: initiate_repl_set.yml diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 new file mode 100644 index 00000000..ed8c8fa4 --- /dev/null +++ b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 @@ -0,0 +1,30 @@ +rs.initiate({ + "_id": "{{ replica_set_name }}", + "members": [ + { + "_id": 0, + "host": "{{ hostvars[ansible_hostname][mongodb_docker_name + (docker_cluster_size|int - 1)|string] }}:{{ mongodb_port }}" + } + ] +}); +sleep(5000); +{% for docker in range(0, docker_cluster_size|int, 1) %} +{%- if docker != (docker_cluster_size|int - 1) -%} +rs.add("{{ hostvars[ansible_hostname][mongodb_docker_name + docker|string] }}:{{ mongodb_port }}"); +{% endif %} +{%- endfor -%} +use admin; +db.createUser(db.createUser({ + "user": "{{ mongodb_admin_user }}", + "pwd": "{{ mongodb_admin_password }}", + "roles": [ + { + "role": "userAdminAnyDatabase", + "db": "admin" + }, + { + "role": "clusterManager", + "db": "admin" + } + ] +}); diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 new file mode 100644 index 00000000..42bca2a3 --- /dev/null +++ b/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 @@ -0,0 +1,7 @@ +rs.initiate( { _id : "{{ replica_set_name }}", members: [ { _id : 0, host :"{{ bdb_hosts[bdb_hosts|length-1]['name'] }}:{{ mongodb_port }}" } ] } ) +sleep(5000); +{% for host in bdb_hosts %} +{%- if ansible_hostname != host["name"] -%} +rs.add("{{ host["name"] }}:{{ mongodb_port }}"); +{% endif %} +{%- endfor -%} \ No newline at end of file diff --git a/pkg/configuration/vars/bdb-config.yml b/pkg/configuration/vars/bdb-config.yml new file mode 100644 index 00000000..f34cd7c7 --- /dev/null +++ b/pkg/configuration/vars/bdb-config.yml @@ -0,0 +1,13 @@ +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.20" + type: "private_network" diff --git a/pkg/scripts/bootstrap_helper.sh b/pkg/scripts/bootstrap_helper.sh index 2c8abd32..8afb4d7e 100755 --- a/pkg/scripts/bootstrap_helper.sh +++ b/pkg/scripts/bootstrap_helper.sh @@ -28,7 +28,7 @@ validate_os_version_and_deps(){ fi ;; fedora) - dnf install bc -y > /dev/null 2>&1 + dnf install bc python2-dnf libselinux-python -y > /dev/null 2>&1 if [[ ($(echo $3 | bc) > $MINIMUM_FEDORA_VERSION) || ($(echo $3 | bc) == $MINIMUM_FEDORA_VERSION) ]]; then rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 @@ -71,20 +71,22 @@ install_dependencies() { # multiple dependencies) install_deps_deb() { echo "Installing Dependencies..." + apt-get update -y apt-get install -y software-properties-common apt-add-repository ppa:ansible/ansible - apt-get update - apt-get install -y --force-yes ansible + apt-get update -y + apt-get install -y "${OS_DEPENDENCIES[@]}" } install_deps_centos() { echo "Installing Dependencies..." yum install epel-release -y yum install -y https://centos7.iuscommunity.org/ius-release.rpm - yum install ansible -y + yum install "${OS_DEPENDENCIES[@]}" -y } install_deps_fedora() { echo "Installing Dependencies..." export LC_ALL=C dnf makecache - dnf -y install ansible python2-dnf -} \ No newline at end of file + echo "${OS_DEPENDENCIES[@]}" + dnf -y install "${OS_DEPENDENCIES[@]}" +} diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py index e80ad0e2..8c064696 100644 --- a/tests/common/test_schema.py +++ b/tests/common/test_schema.py @@ -11,7 +11,7 @@ from pytest import raises from bigchaindb.common.exceptions import SchemaValidationError from bigchaindb.common.schema import ( - TX_SCHEMA_COMMON, VOTE_SCHEMA, drop_schema_descriptions, + TX_SCHEMA_COMMON, VOTE_SCHEMA, validate_transaction_schema, validate_vote_schema) SUPPORTED_CRYPTOCONDITION_TYPES = ('threshold-sha-256', 'ed25519-sha-256') @@ -46,49 +46,6 @@ def test_vote_schema_additionalproperties(): _test_additionalproperties(VOTE_SCHEMA) -def test_drop_descriptions(): - node = { - 'description': 'abc', - 'properties': { - 'description': { - 'description': ('The property named "description" should stay' - 'but description meta field goes'), - }, - 'properties': { - 'description': 'this must go' - }, - 'any': { - 'anyOf': [ - { - 'description': 'must go' - } - ] - } - }, - 'definitions': { - 'wat': { - 'description': 'go' - } - } - } - expected = { - 'properties': { - 'description': {}, - 'properties': {}, - 'any': { - 'anyOf': [ - {} - ] - } - }, - 'definitions': { - 'wat': {}, - } - } - drop_schema_descriptions(node) - assert node == expected - - ################################################################################ # Test call transaction schema diff --git a/tests/integration/test_federation.py b/tests/integration/test_federation.py index 22e2e8da..05d8f32e 100644 --- a/tests/integration/test_federation.py +++ b/tests/integration/test_federation.py @@ -116,8 +116,8 @@ def test_elect_valid(federation_3): @pytest.mark.bdb -@pytest.mark.skip_travis_rdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_invalid(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 64035c78..0c67b783 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -5,7 +5,7 @@ import pytest pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('processes')] -@pytest.mark.skip_travis_rdb +@pytest.mark.serial def test_double_create(b, user_pk): from bigchaindb.models import Transaction from bigchaindb.backend.query import count_blocks