From f5a32e35c5f2bdea89fcf79cb821e6f694b050f7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 23 Mar 2017 18:21:55 +0100 Subject: [PATCH 1/8] docs: 1st draft of page about updating all s/w on a BDB node on k8s --- .../cloud-deployment-templates/index.rst | 2 + .../upgrade-on-kubernetes.rst | 105 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 837dc66d..28ac7923 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -16,3 +16,5 @@ If you find the cloud deployment templates for nodes helpful, then you may also template-kubernetes-azure node-on-kubernetes add-node-on-kubernetes + upgrade-on-kubernetes + \ No newline at end of file diff --git a/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst new file mode 100644 index 00000000..348abf22 --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst @@ -0,0 +1,105 @@ +Kubernetes Template: Upgrade all Software in a BigchainDB Node +============================================================== + +This page outlines how to upgrade all the software associated +with a BigchainDB node running on Kubernetes, +including host operating systems, Docker, Kubernetes, +and BigchainDB-related software. + + +Upgrade Host OS, Docker and Kubernetes +-------------------------------------- + +Some Kubernetes installation & management systems +can do full or partial upgrades of host OSes, Docker, +or Kubernetes, e.g. +`Tectonic `_, +`Rancher `_, +and +`Kubo `_. +Consult the documentation for your system. + +**Azure Container Service (ACS).** +On Dec. 15, 2016, a Microsoft employee +`wrote `_: +"In the coming months we [the Azure Kubernetes team] will be building managed updates in the ACS service." +At the time of writing, managed updates were not yet available, +but you should check the latest +`ACS documentation `_ +to see what's available now. +Also at the time of writing, ACS only supported Ubuntu +as the host (master and agent) operating system. +You can upgrade Ubuntu and Docker on Azure +by SSHing into each of the hosts, +as documented on +:ref:`another page `. + +In general, you can SSH to each host in your Kubernetes Cluster +to update the OS and Docker. + +.. note:: + + Once you are in an SSH session with a host, + the ``docker info`` command is a handy way to detemine the + host OS (including version) and the Docker version. + +When you want to upgrade the software on a Kubernetes node, +you should "drain" the node first, +i.e. tell Kubernetes to gracefully terminate all pods +on the node and mark it as unscheduleable +(so no new pods get put on the node during its downtime). + +.. code:: + + kubectl drain $NODENAME + +There are `more details in the Kubernetes docs `_, +including instructions to make the node scheduleable again. + +To manually upgrade the host OS, +see the docs for that OS. + +To manually upgrade Docker, see +`the Docker docs `_. + +To manually upgrade all Kubernetes software in your Kubernetes cluster, see +`the Kubernetes docs `_. + + +Upgrade BigchainDB-Related Software +----------------------------------- + +We use Kubernetes "Deployments" for NGINX, BigchainDB, +and most other BigchainDB-related software. +The only exception is MongoDB; we use a Kubernetes +StatefulSet for that. + +The nice thing about Kubernetes Deployments +is that Kubernetes can manage most of the upgrade process. +A typical upgrade workflow for a single Deployment would be: + +.. code:: + + $ KUBE_EDITOR=nano kubectl edit deployment/ + +The `kubectl edit `_ +command opens the specified editor (nano in the above example), +allowing you to edit the specified Deployment *in the Kubernetes cluster*. +You can change the version tag on the Docker image, for example. +Don't forget to save your edits before exiting the editor. +The Kubernetes docs have more information about +`updating a Deployment `_. + + +The upgrade story for the MongoDB StatefulSet is *different*. +(This is because MongoDB has persistent state, +which is stored in some storage associated with a PersistentVolumeClaim.) +At the time of writing, StatefulSets were still in beta, +and they did not support automated image upgrade (Docker image tag upgrade). +We expect that to change. +Rather than trying to keep these docs up-to-date, +we advise you to check out the current +`Kubernetes docs about updating containers in StatefulSets +`_. + + From c6de90fa79c080bb1403260b36f873e0ed1b4a69 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 29 Mar 2017 09:56:41 +0200 Subject: [PATCH 2/8] Upgrade rapidjson to latest 0.0.11 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a38bb1f..5fb201f4 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ install_requires = [ 'pymongo~=3.4', 'pysha3~=1.0.2', 'cryptoconditions>=0.5.0', - 'python-rapidjson==0.0.8', + 'python-rapidjson==0.0.11', 'logstats>=0.2.1', 'flask>=0.10.1', 'flask-restful~=0.3.0', From ead832a130ed2136dac7735e502af5b708468b3a Mon Sep 17 00:00:00 2001 From: morrme Date: Mon, 27 Mar 2017 05:13:29 -0500 Subject: [PATCH 3/8] added Python 3.6 per issue #1331 --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index da7ae05f..e558d154 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ cache: pip python: - 3.4 - 3.5 - + - 3.6 + env: - TOXENV=flake8 - TOXENV=docsroot @@ -30,6 +31,12 @@ matrix: env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb - python: 3.5 env: BIGCHAINDB_DATABASE_BACKEND=mongodb + - python: 3.6 + addons: + rethinkdb: '2.3.5' + env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb + - python: 3.6 + env: BIGCHAINDB_DATABASE_BACKEND=mongodb before_install: sudo .ci/travis-before-install.sh From 5c2bab078fac8bdd65a3ec77a8e524084fff0c70 Mon Sep 17 00:00:00 2001 From: morrme Date: Mon, 27 Mar 2017 05:14:40 -0500 Subject: [PATCH 4/8] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d2cd2a2c..8f299471 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = true -envlist = py{34,35}-{rethinkdb,mongodb}, flake8, docsroot, docsserver +envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver [base] basepython = python3.5 From 64ef0dd9a17e70b3e02b7a2c5f67eda03073e13d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 31 Mar 2017 10:54:02 +0200 Subject: [PATCH 5/8] Test docs building and flake8 only for Python 3.6 --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e558d154..9fc4e278 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,17 @@ env: matrix: fast_finish: true exclude: - - python: 3.4 + - python: 3.4 env: TOXENV=flake8 - - python: 3.4 + - python: 3.4 env: TOXENV=docsroot - - python: 3.4 + - python: 3.4 + env: TOXENV=docsserver + - python: 3.5 + env: TOXENV=flake8 + - python: 3.5 + env: TOXENV=docsroot + - python: 3.5 env: TOXENV=docsserver include: - python: 3.4 From 36edbc5f354e1b5b62af03246e60e546951f82a8 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 31 Mar 2017 10:54:59 +0200 Subject: [PATCH 6/8] Set base version for Python to 3.6 in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8f299471..bdaea034 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ skipsdist = true envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver [base] -basepython = python3.5 +basepython = python3.6 deps = pip>=9.0.1 [testenv] From 6fb793e52f716a356579fc60349840fe6e2bd282 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 29 Mar 2017 10:22:15 +0200 Subject: [PATCH 7/8] Update changelog for 0.9.5 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9af7ccc3..2148903b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [0.9.5] - 2017-03-29 +Tag name: v0.9.5 + +### Fixed +Upgrade `python-rapidjson` to `0.0.11`(fixes #1350 - thanks to @ferOnti for +reporting). + ## [0.9.4] - 2017-03-16 Tag name: v0.9.4 From 87eb070ed68533abd42f736b3ac293e0c6883416 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 3 Mar 2017 01:47:08 +0100 Subject: [PATCH 8/8] Refactor core.BigchainDB.get_outputs --- bigchaindb/core.py | 64 +++++++++++++++++++++++++-------------------- bigchaindb/utils.py | 13 +++++++++ 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index a9143f33..e6783a6d 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -19,14 +19,17 @@ class Bigchain(object): Create, read, sign, write transactions to the database """ - # return if a block has been voted invalid BLOCK_INVALID = 'invalid' - # return if a block is valid, or tx is in valid block + """return if a block has been voted invalid""" + BLOCK_VALID = TX_VALID = 'valid' - # return if block is undecided, or tx is in undecided block + """return if a block is valid, or tx is in valid block""" + BLOCK_UNDECIDED = TX_UNDECIDED = 'undecided' - # return if transaction is in backlog + """return if block is undecided, or tx is in undecided block""" + TX_IN_BACKLOG = 'backlog' + """return if transaction is in backlog""" def __init__(self, public_key=None, private_key=None, keyring=[], connection=None, backlog_reassign_delay=None): """Initialize the Bigchain instance @@ -372,32 +375,37 @@ class Bigchain(object): """ # get all transactions in which owner is in the `owners_after` list response = backend.query.get_owned_ids(self.connection, owner) - links = [] + return [ + TransactionLink(tx['id'], index) + for tx in response + if not self.is_tx_strictly_in_invalid_block(tx['id']) + for index, output in enumerate(tx['outputs']) + if utils.output_has_owner(output, owner) + ] - for tx in response: - # disregard transactions from invalid blocks - validity = self.get_blocks_status_containing_tx(tx['id']) - if Bigchain.BLOCK_VALID not in validity.values(): - if Bigchain.BLOCK_UNDECIDED not in validity.values(): - continue + def is_tx_strictly_in_invalid_block(self, txid): + """ + Checks whether the transaction with the given ``txid`` + *strictly* belongs to an invalid block. - # NOTE: It's OK to not serialize the transaction here, as we do not - # use it after the execution of this function. - # a transaction can contain multiple outputs so we need to iterate over all of them - # to get a list of outputs available to spend - for index, output in enumerate(tx['outputs']): - # for simple signature conditions there are no subfulfillments - # check if the owner is in the condition `owners_after` - if len(output['public_keys']) == 1: - if output['condition']['details']['public_key'] == owner: - links.append(TransactionLink(tx['id'], index)) - else: - # for transactions with multiple `public_keys` there will be several subfulfillments nested - # in the condition. We need to iterate the subfulfillments to make sure there is a - # subfulfillment for `owner` - if utils.condition_details_has_owner(output['condition']['details'], owner): - links.append(TransactionLink(tx['id'], index)) - return links + Args: + txid (str): Transaction id. + + Returns: + bool: ``True`` if the transaction *strictly* belongs to a + block that is invalid. ``False`` otherwise. + + Note: + Since a transaction may be in multiple blocks, with + different statuses, the term "strictly" is used to + emphasize that if a transaction is said to be in an invalid + block, it means that it is not in any other block that is + either valid or undecided. + + """ + validity = self.get_blocks_status_containing_tx(txid) + return (Bigchain.BLOCK_VALID not in validity.values() and + Bigchain.BLOCK_UNDECIDED not in validity.values()) def get_owned_ids(self, owner): """Retrieve a list of ``txid`` s that can be used as inputs. diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 4d7177d9..f87916b7 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -113,6 +113,19 @@ def condition_details_has_owner(condition_details, owner): return False +def output_has_owner(output, owner): + # TODO + # Check whether it is really necessary to treat the single key case + # differently from the multiple keys case, and why not just use the same + # function for both cases. + if len(output['public_keys']) > 1: + return condition_details_has_owner( + output['condition']['details'], owner) + elif len(output['public_keys']) == 1: + return output['condition']['details']['public_key'] == owner + # TODO raise proper exception, e.g. invalid tx payload? + + def is_genesis_block(block): """Check if the block is the genesis block.