Merge remote-tracking branch 'origin/master' into feat/905/abstract-db-changefeed

Makefile

@@ -0,0 +1,88 @@
+.PHONY: clean clean-test clean-pyc clean-build docs help
+.DEFAULT_GOAL := help
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+try:
+	from urllib import pathname2url
+except:
+	from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+
+define PRINT_HELP_PYSCRIPT
+import re, sys
+
+for line in sys.stdin:
+	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+	if match:
+		target, help = match.groups()
+		print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+help:
+	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+
+clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+
+
+clean-build: ## remove build artifacts
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: ## remove Python file artifacts
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## remove test and coverage artifacts
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+
+lint: ## check style with flake8
+	flake8 bigchaindb tests
+
+test: ## run tests quickly with the default Python
+	py.test
+	
+
+test-all: ## run tests on every Python version with tox
+	tox
+
+coverage: ## check code coverage quickly with the default Python
+	coverage run --source bigchaindb py.test
+	
+		coverage report -m
+		coverage html
+		$(BROWSER) htmlcov/index.html
+
+docs: ## generate Sphinx HTML documentation, including API docs
+	rm -f docs/bigchaindb.rst
+	rm -f docs/modules.rst
+	sphinx-apidoc -o docs/ bigchaindb
+	$(MAKE) -C docs clean
+	$(MAKE) -C docs html
+	$(BROWSER) docs/_build/html/index.html
+
+servedocs: docs ## compile the docs watching for changes
+	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
+
+release: clean ## package and upload a release
+	python setup.py sdist upload
+	python setup.py bdist_wheel upload
+
+dist: clean ## builds source and wheel package
+	python setup.py sdist
+	python setup.py bdist_wheel
+	ls -l dist
+
+install: clean ## install the package to the active Python's site-packages
+	python setup.py install

bigchaindb/backend/connection.py

@@ -1,6 +1,7 @@
+from importlib import import_module
+
 import bigchaindb
 from bigchaindb.common.exceptions import ConfigurationError
-from importlib import import_module
 
 
 BACKENDS = {

bigchaindb/pipelines/block.py

@@ -6,6 +6,7 @@ function.
 """
 
 import logging
+
 from multipipes import Pipeline, Node, Pipe
 
 import bigchaindb
@@ -135,14 +136,6 @@ class BlockPipeline:
         return block
 
 
-def initial():
-    """Return old transactions from the backlog."""
-
-    bigchain = Bigchain()
-
-    return bigchain.get_old_transactions()
-
-
 def create_pipeline():
     """Create and return the pipeline of operations to be distributed
     on different processes."""

docker-compose.yml

@@ -28,6 +28,7 @@ services:
       - ./setup.cfg:/usr/src/app/setup.cfg
       - ./pytest.ini:/usr/src/app/pytest.ini
       - ./tox.ini:/usr/src/app/tox.ini
+      - ./Makefile:/usr/src/app/Makefile
     environment:
       BIGCHAINDB_DATABASE_HOST: rdb
       BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984

docs/root/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -W
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build

docs/server/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -W
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build

docs/server/source/appendices/cryptography.md

@@ -8,31 +8,69 @@ as described in [the section on JSON serialization](json-serialization.html).
 
 ## Hashes
 
-We compute hashes using the SHA3-256 algorithm and
-[pysha3](https://bitbucket.org/tiran/pykeccak) as the Python implementation. We
-store the hex-encoded hash in the database. For example:
+BigchainDB computes transaction and block hashes using an implementation of the
+[SHA3-256](https://en.wikipedia.org/wiki/SHA-3)
+algorithm provided by the
+[**pysha3** package](https://bitbucket.org/tiran/pykeccak),
+which is a wrapper around the optimized reference implementation
+from [http://keccak.noekeon.org](http://keccak.noekeon.org).
+
+Here's the relevant code from `bigchaindb/bigchaindb/common/crypto.py`
+(as of 11 December 2016):
 
 ```python
-import hashlib
-# monkey patch hashlib with sha3 functions
 import sha3
 
-data = "message"
-tx_hash = hashlib.sha3_256(data).hexdigest()
+def hash_data(data):
+    """Hash the provided data using SHA3-256"""
+    return sha3.sha3_256(data.encode()).hexdigest()
 ```
 
+The incoming `data` is understood to be a Python 3 string,
+which may contain Unicode characters such as `'ü'` or `'字'`.
+The Python 3 `encode()` method converts `data` to a bytes object.
+`sha3.sha3_256(data.encode())` is a _sha3.SHA3 object;
+the `hexdigest()` method converts it to a hexadecimal string.
+For example:
+
+```python
+>>> import sha3
+>>> data = '字'
+>>> sha3.sha3_256(data.encode()).hexdigest()
+'c67820de36d949a35ca24492e15767e2972b22f77213f6704ac0adec123c5690'
+```
+
+Note: Hashlocks (which are one kind of crypto-condition)
+may use a different hash function.
+
+
 ## Signature Algorithm and Keys
 
 BigchainDB uses the [Ed25519](https://ed25519.cr.yp.to/) public-key signature
 system for generating its public/private key pairs. Ed25519 is an instance of
 the [Edwards-curve Digital Signature Algorithm
-(EdDSA)](https://en.wikipedia.org/wiki/EdDSA). As of April 2016, EdDSA was in
-["Internet-Draft" status with the
-IETF](https://tools.ietf.org/html/draft-irtf-cfrg-eddsa-05) but was [already
+(EdDSA)](https://en.wikipedia.org/wiki/EdDSA). As of December 2016, EdDSA was an
+["Internet-Draft" with the
+IETF](https://tools.ietf.org/html/draft-irtf-cfrg-eddsa-08) but was [already
 widely used](https://ianix.com/pub/ed25519-deployment.html).
 
-BigchainDB uses the the [ed25519](https://github.com/warner/python-ed25519)
-Python package, overloaded by the [cryptoconditions
-library](https://github.com/bigchaindb/cryptoconditions).
+BigchainDB uses the the 
+[**cryptoconditions** package](https://github.com/bigchaindb/cryptoconditions)
+to do signature and keypair-related calculations.
+That package, in turn, uses the [**PyNaCl** package](https://pypi.python.org/pypi/PyNaCl),
+a Python binding to the Networking and Cryptography (NaCl) library.
 
-All keys are represented with the base58 encoding by default.
+All keys are represented with
+[a Base58 encoding](https://en.wikipedia.org/wiki/Base58).
+The cryptoconditions package uses the
+[**base58** package](https://pypi.python.org/pypi/base58)
+to calculate a Base58 encoding.
+(There's no standard for Base58 encoding.)
+Here's an example public/private key pair:
+
+```js
+"keypair": {
+    "public": "9WYFf8T65bv4S8jKU8wongKPD4AmMZAwvk1absFDbYLM",
+    "private": "3x7MQpPq8AEUGEuzAxSVHjU1FhLWVQJKFNNkvHhJPGCX"
+}
+```

setup.py

@@ -30,6 +30,7 @@ check_setuptools_features()
 dev_require = [
     'ipdb',
     'ipython',
+    'watchdog',
 ]
 
 docs_require = [
@@ -63,7 +64,7 @@ install_requires = [
     'pysha3>=0.3',
     'cryptoconditions>=0.5.0',
     'statsd>=3.2.1',
-    'python-rapidjson>=0.0.6',
+    'python-rapidjson>=0.0.8',
     'logstats>=0.2.1',
     'flask>=0.10.1',
     'flask-restful~=0.3.0',
@@ -115,5 +116,5 @@ setup(
         'dev': dev_require + tests_require + docs_require + benchmarks_require,
         'docs': docs_require,
     },
-    package_data={'bigchaindb.common.schema': ['transaction.yaml']},
+    package_data={'bigchaindb.common.schema': ['*.yaml']},
 )

tests/pipelines/test_block_creation.py

@@ -131,22 +131,6 @@ def test_delete_tx(b, user_pk):
         assert status != b.TX_IN_BACKLOG
 
 
-def test_prefeed(b, user_pk):
-    import random
-    from bigchaindb.models import Transaction
-    from bigchaindb.pipelines.block import initial
-
-    for i in range(100):
-        tx = Transaction.create([b.me], [([user_pk], 1)],
-                                {'msg': random.random()})
-        tx = tx.sign([b.me_private])
-        b.write_transaction(tx)
-
-    backlog = initial()
-
-    assert len(list(backlog)) == 100
-
-
 @patch('bigchaindb.pipelines.block.create_pipeline')
 def test_start(create_pipeline):
     from bigchaindb.pipelines import block
@@ -165,28 +149,39 @@ def test_full_pipeline(b, user_pk):
     from bigchaindb.pipelines.block import create_pipeline, get_changefeed
 
     outpipe = Pipe()
+
+    pipeline = create_pipeline()
+    pipeline.setup(outdata=outpipe)
+    inpipe = pipeline.items[0]
+
     # include myself here, so that some tx are actually assigned to me
     b.nodes_except_me = [b.me, 'aaa', 'bbb', 'ccc']
+    number_assigned_to_others = 0
     for i in range(100):
         tx = Transaction.create([b.me], [([user_pk], 1)],
                                 {'msg': random.random()})
         tx = tx.sign([b.me_private])
 
-        b.write_transaction(tx)
+        tx = tx.to_dict()
 
-    assert query.count_backlog(b.connection) == 100
+        # simulate write_transaction
+        tx['assignee'] = random.choice(b.nodes_except_me)
+        if tx['assignee'] != b.me:
+            number_assigned_to_others += 1
+        tx['assignment_timestamp'] = time.time()
+        inpipe.put(tx)
+
+    assert inpipe.qsize() == 100
 
-    pipeline = create_pipeline()
-    pipeline.setup(indata=get_changefeed(), outdata=outpipe)
     pipeline.start()
 
     time.sleep(2)
-    pipeline.terminate()
 
+    pipeline.terminate()
     block_doc = outpipe.get()
     chained_block = b.get_block(block_doc.id)
     chained_block = Block.from_dict(chained_block)
 
     block_len = len(block_doc.transactions)
     assert chained_block == block_doc
-    assert query.count_backlog(b.connection) == 100 - block_len
+    assert number_assigned_to_others == 100 - block_len

tests/pipelines/test_election.py

@@ -118,8 +118,8 @@ def test_check_requeue_transaction(b, user_pk):
 
     e.requeue_transactions(test_block)
 
+    time.sleep(1)
     backlog_tx, status = b.get_transaction(tx1.id, include_status=True)
-    #backlog_tx = b.connection.run(r.table('backlog').get(tx1.id))
     assert status == b.TX_IN_BACKLOG
     assert backlog_tx == tx1