Merge remote-tracking branch 'upstream/master' into bug/437/pretty-msg-drop-nonexistent-db

.travis.yml

@@ -4,6 +4,7 @@ cache: pip
 python:
   - 3.4
   - 3.5
+  - 3.6
   
 env:
   - TOXENV=flake8
@@ -19,6 +20,12 @@ matrix:
     env: TOXENV=docsroot
   - python: 3.4
     env: TOXENV=docsserver
+  - python: 3.5
+    env: TOXENV=flake8 
+  - python: 3.5 
+    env: TOXENV=docsroot
+  - python: 3.5 
+    env: TOXENV=docsserver
   include:
   - python: 3.4
     addons: 
@@ -30,6 +37,12 @@ matrix:
     env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb
   - python: 3.5
     env: BIGCHAINDB_DATABASE_BACKEND=mongodb
+  - python: 3.6
+    addons: 
+      rethinkdb: '2.3.5'
+    env: BIGCHAINDB_DATABASE_BACKEND=rethinkdb
+  - python: 3.6
+    env: BIGCHAINDB_DATABASE_BACKEND=mongodb
 
 before_install: sudo .ci/travis-before-install.sh
 

CHANGELOG.md

@@ -15,6 +15,13 @@ For reference, the possible headings are:
 * **External Contributors** to list contributors outside of BigchainDB GmbH.
 * **Notes**
 
+## [0.9.5] - 2017-03-29
+Tag name: v0.9.5
+
+### Fixed
+Upgrade `python-rapidjson` to `0.0.11`(fixes #1350 - thanks to @ferOnti for
+reporting).
+
 ## [0.9.4] - 2017-03-16
 Tag name: v0.9.4
 

bigchaindb/core.py

@@ -19,14 +19,17 @@ class Bigchain(object):
     Create, read, sign, write transactions to the database
     """
 
-    # return if a block has been voted invalid
     BLOCK_INVALID = 'invalid'
-    # return if a block is valid, or tx is in valid block
+    """return if a block has been voted invalid"""
+
     BLOCK_VALID = TX_VALID = 'valid'
-    # return if block is undecided, or tx is in undecided block
+    """return if a block is valid, or tx is in valid block"""
+
     BLOCK_UNDECIDED = TX_UNDECIDED = 'undecided'
-    # return if transaction is in backlog
+    """return if block is undecided, or tx is in undecided block"""
+
     TX_IN_BACKLOG = 'backlog'
+    """return if transaction is in backlog"""
 
     def __init__(self, public_key=None, private_key=None, keyring=[], connection=None, backlog_reassign_delay=None):
         """Initialize the Bigchain instance
@@ -372,32 +375,37 @@ class Bigchain(object):
         """
         # get all transactions in which owner is in the `owners_after` list
         response = backend.query.get_owned_ids(self.connection, owner)
-        links = []
+        return [
+            TransactionLink(tx['id'], index)
+            for tx in response
+            if not self.is_tx_strictly_in_invalid_block(tx['id'])
+            for index, output in enumerate(tx['outputs'])
+            if utils.output_has_owner(output, owner)
+        ]
 
-        for tx in response:
-            # disregard transactions from invalid blocks
-            validity = self.get_blocks_status_containing_tx(tx['id'])
-            if Bigchain.BLOCK_VALID not in validity.values():
-                if Bigchain.BLOCK_UNDECIDED not in validity.values():
-                    continue
+    def is_tx_strictly_in_invalid_block(self, txid):
+        """
+        Checks whether the transaction with the given ``txid``
+        *strictly* belongs to an invalid block.
 
-            # NOTE: It's OK to not serialize the transaction here, as we do not
-            # use it after the execution of this function.
-            # a transaction can contain multiple outputs so we need to iterate over all of them
-            # to get a list of outputs available to spend
-            for index, output in enumerate(tx['outputs']):
-                # for simple signature conditions there are no subfulfillments
-                # check if the owner is in the condition `owners_after`
-                if len(output['public_keys']) == 1:
-                    if output['condition']['details']['public_key'] == owner:
-                        links.append(TransactionLink(tx['id'], index))
-                else:
-                    # for transactions with multiple `public_keys` there will be several subfulfillments nested
-                    # in the condition. We need to iterate the subfulfillments to make sure there is a
-                    # subfulfillment for `owner`
-                    if utils.condition_details_has_owner(output['condition']['details'], owner):
-                        links.append(TransactionLink(tx['id'], index))
-        return links
+        Args:
+            txid (str): Transaction id.
+
+        Returns:
+            bool: ``True`` if the transaction *strictly* belongs to a
+            block that is invalid. ``False`` otherwise.
+
+        Note:
+            Since a transaction may be in multiple blocks, with
+            different statuses, the term "strictly" is used to
+            emphasize that if a transaction is said to be in an invalid
+            block, it means that it is not in any other block that is
+            either valid or undecided.
+
+        """
+        validity = self.get_blocks_status_containing_tx(txid)
+        return (Bigchain.BLOCK_VALID not in validity.values() and
+                Bigchain.BLOCK_UNDECIDED not in validity.values())
 
     def get_owned_ids(self, owner):
         """Retrieve a list of ``txid`` s that can be used as inputs.

bigchaindb/utils.py

@@ -113,6 +113,19 @@ def condition_details_has_owner(condition_details, owner):
     return False
 
 
+def output_has_owner(output, owner):
+    # TODO
+    # Check whether it is really necessary to treat the single key case
+    # differently from the multiple keys case, and why not just use the same
+    # function for both cases.
+    if len(output['public_keys']) > 1:
+        return condition_details_has_owner(
+            output['condition']['details'], owner)
+    elif len(output['public_keys']) == 1:
+        return output['condition']['details']['public_key'] == owner
+    # TODO raise proper exception, e.g. invalid tx payload?
+
+
 def is_genesis_block(block):
     """Check if the block is the genesis block.
 

docs/server/source/cloud-deployment-templates/index.rst

@@ -16,3 +16,5 @@ If you find the cloud deployment templates for nodes helpful, then you may also
    template-kubernetes-azure
    node-on-kubernetes
    add-node-on-kubernetes
+   upgrade-on-kubernetes
+   

docs/server/source/cloud-deployment-templates/upgrade-on-kubernetes.rst

@@ -0,0 +1,105 @@
+Kubernetes Template: Upgrade all Software in a BigchainDB Node
+==============================================================
+
+This page outlines how to upgrade all the software associated
+with a BigchainDB node running on Kubernetes,
+including host operating systems, Docker, Kubernetes,
+and BigchainDB-related software.
+
+
+Upgrade Host OS, Docker and Kubernetes
+--------------------------------------
+
+Some Kubernetes installation & management systems
+can do full or partial upgrades of host OSes, Docker,
+or Kubernetes, e.g.
+`Tectonic <https://coreos.com/tectonic/>`_, 
+`Rancher <https://docs.rancher.com/rancher/v1.5/en/>`_,
+and 
+`Kubo <https://pivotal.io/kubo>`_.
+Consult the documentation for your system.
+
+**Azure Container Service (ACS).**
+On Dec. 15, 2016, a Microsoft employee
+`wrote <https://github.com/colemickens/azure-kubernetes-status/issues/15#issuecomment-267453251>`_:
+"In the coming months we [the Azure Kubernetes team] will be building managed updates in the ACS service."
+At the time of writing, managed updates were not yet available,
+but you should check the latest
+`ACS documentation <https://docs.microsoft.com/en-us/azure/container-service/>`_
+to see what's available now.
+Also at the time of writing, ACS only supported Ubuntu
+as the host (master and agent) operating system.
+You can upgrade Ubuntu and Docker on Azure
+by SSHing into each of the hosts,
+as documented on 
+:ref:`another page <Optional: SSH to Your New Kubernetes Cluster Nodes>`.
+
+In general, you can SSH to each host in your Kubernetes Cluster
+to update the OS and Docker.
+
+.. note::
+
+   Once you are in an SSH session with a host,
+   the ``docker info`` command is a handy way to detemine the
+   host OS (including version) and the Docker version.
+
+When you want to upgrade the software on a Kubernetes node,
+you should "drain" the node first,
+i.e. tell Kubernetes to gracefully terminate all pods
+on the node and mark it as unscheduleable
+(so no new pods get put on the node during its downtime).
+
+.. code::
+
+   kubectl drain $NODENAME
+
+There are `more details in the Kubernetes docs <https://kubernetes.io/docs/admin/cluster-management/#maintenance-on-a-node>`_,
+including instructions to make the node scheduleable again.
+
+To manually upgrade the host OS,
+see the docs for that OS.
+
+To manually upgrade Docker, see
+`the Docker docs <https://docs.docker.com/>`_.
+
+To manually upgrade all Kubernetes software in your Kubernetes cluster, see
+`the Kubernetes docs <https://kubernetes.io/docs/admin/cluster-management/>`_.
+
+
+Upgrade BigchainDB-Related Software
+-----------------------------------
+
+We use Kubernetes "Deployments" for NGINX, BigchainDB,
+and most other BigchainDB-related software.
+The only exception is MongoDB; we use a Kubernetes
+StatefulSet for that.
+
+The nice thing about Kubernetes Deployments
+is that Kubernetes can manage most of the upgrade process.
+A typical upgrade workflow for a single Deployment would be:
+
+.. code::
+
+   $ KUBE_EDITOR=nano kubectl edit deployment/<name of Deployment>
+
+The `kubectl edit <https://kubernetes.io/docs/user-guide/kubectl/kubectl_edit/>`_ 
+command opens the specified editor (nano in the above example),
+allowing you to edit the specified Deployment *in the Kubernetes cluster*.
+You can change the version tag on the Docker image, for example. 
+Don't forget to save your edits before exiting the editor.
+The Kubernetes docs have more information about
+`updating a Deployment <https://kubernetes.io/docs/user-guide/deployments/#updating-a-deployment>`_.
+
+
+The upgrade story for the MongoDB StatefulSet is *different*.
+(This is because MongoDB has persistent state,
+which is stored in some storage associated with a PersistentVolumeClaim.)
+At the time of writing, StatefulSets were still in beta,
+and they did not support automated image upgrade (Docker image tag upgrade).
+We expect that to change.
+Rather than trying to keep these docs up-to-date,
+we advise you to check out the current
+`Kubernetes docs about updating containers in StatefulSets
+<https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-containers>`_.
+
+

setup.py

@@ -67,7 +67,7 @@ install_requires = [
     'pymongo~=3.4',
     'pysha3~=1.0.2',
     'cryptoconditions>=0.5.0',
-    'python-rapidjson==0.0.8',
+    'python-rapidjson==0.0.11',
     'logstats>=0.2.1',
     'flask>=0.10.1',
     'flask-restful~=0.3.0',

tox.ini

@@ -1,9 +1,9 @@
 [tox]
 skipsdist = true
-envlist = py{34,35}-{rethinkdb,mongodb}, flake8, docsroot, docsserver
+envlist = py{34,35,36}-{rethinkdb,mongodb}, flake8, docsroot, docsserver
 
 [base]
-basepython = python3.5
+basepython = python3.6
 deps = pip>=9.0.1   
 
 [testenv]