Abstract db layer cherrypick docs (#932)

* Add README.md to tests/ to describe test structure and pytest customizations

* Add package-level docstrings to backends

* Add README.md explaining the backend architecture

* Small improvements to docstrings in backend modules

* Restructure the backend automodule docs

* Add more docstrings to backend connections

* Add init to base backend Connection class to document expected interface

* Reword the backend/README.md docs

bigchaindb/backend/README.md

@@ -0,0 +1,45 @@
+# Backend Interfaces
+
+## Structure
+
+- [`changefeed.py`](./changefeed.py): Changefeed-related interfaces
+- [`connection.py`](./connection.py): Database connection-related interfaces
+- [`query.py`](./query.py): Database query-related interfaces, dispatched through single-dispatch
+- [`schema.py`](./schema.py): Database setup and schema-related interfaces, dispatched through
+  single-dispatch
+
+Built-in implementations (e.g. [RethinkDB's](./rethinkdb)) are provided in sub-directories and
+have their connection type's location exposed as `BACKENDS` in [`connection.py`](./connection.py).
+
+## Single-Dispatched Interfaces
+
+The architecture of this module is based heavily upon Python's newly-introduced [single-dispatch
+generic functions](https://www.python.org/dev/peps/pep-0443/). Single-dispatch is convenient,
+because it allows Python, rather than something we design ourselves, to manage the dispatching of
+generic functions based on certain conditions being met (e.g. the database backend to use).
+
+To see what this looks like in BigchainDB, first note that our backend interfaces have been
+configured to dispatch based on a backend's **connection type**.
+
+Call `bigchaindb.backend.connect()` to create an instance of a `Connection`:
+
+```python
+from bigchaindb.backend import connect
+connection = connect()  # By default, uses the current configuration for backend, host, port, etc.
+```
+
+Then, we can call a backend function by directly calling its interface:
+
+```python
+from bigchaindb.backend import query
+query.write_transaction(connection, ...)
+```
+
+Notice that we don't need to care about which backend implementation to use or how to access it.
+Code can simply call the base interface function with a `Connection` instance, and single-dispatch
+will handle routing the call to the actual implementation.
+
+BigchainDB will load and register the configured backend's implementation automatically (see
+`bigchaindb.backend.connect()`), so you should always just be able to call an interface function if
+you have a `Connection` instance. A few helper utilities (see [`backend/utils.py`](./utils.py)) are
+also provided to make registering new backend implementations easier.

bigchaindb/backend/__init__.py

@@ -1,4 +1,10 @@
-"""Backend interfaces ..."""
+"""Generic backend database interfaces expected by BigchainDB.
+
+The interfaces in this module allow BigchainDB to be agnostic about its
+database backend. One can configure BigchainDB to use different databases as
+its data store by setting the ``database.backend`` property in the
+configuration or the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable.
+"""
 
 # Include the backend interfaces
 from bigchaindb.backend import changefeed, schema, query  # noqa

bigchaindb/backend/changefeed.py

@@ -1 +1 @@
-"""Changefeed interfaces for backend databases."""
+"""Changefeed interfaces for backends."""

bigchaindb/backend/connection.py

@@ -9,7 +9,10 @@ BACKENDS = {
 
 
 def connect(backend=None, host=None, port=None, name=None):
-    """Create a connection to the database backend.
+    """Create a new connection to the database backend.
+
+    All arguments default to the current configuration's values if not
+    given.
 
     Args:
         backend (str): the name of the backend to use.
@@ -18,7 +21,12 @@ def connect(backend=None, host=None, port=None, name=None):
         name (str): the name of the database to use.
 
     Returns:
-        An instance of :class:`~bigchaindb.backend.connection.Connection`.
+        An instance of :class:`~bigchaindb.backend.connection.Connection`
+        based on the given (or defaulted) :attr:`backend`.
+
+    Raises:
+        :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend`
+            is not supported or could not be loaded.
     """
 
     backend = backend or bigchaindb.config['database']['backend']
@@ -39,6 +47,28 @@ def connect(backend=None, host=None, port=None, name=None):
 
 
 class Connection:
+    """Connection class interface.
+
+    All backend implementations should provide a connection class that
+    from and implements this class.
+    """
+
+    def __init__(self, host=None, port=None, dbname=None, *args, **kwargs):
+        """Create a new :class:`~.Connection` instance.
+
+        Args:
+            host (str): the host to connect to.
+            port (int): the port to connect to.
+            dbname (str): the name of the database to use.
+            **kwargs: arbitrary keyword arguments provided by the
+                configuration's ``database`` settings
+        """
 
     def run(self, query):
+        """Run a query.
+
+        Args:
+            query: the query to run
+        """
+
         raise NotImplementedError()

bigchaindb/backend/query.py

@@ -1,4 +1,4 @@
-"""Query interfaces for backend databases."""
+"""Query interfaces for backends."""
 
 from functools import singledispatch
 

bigchaindb/backend/rethinkdb/__init__.py

@@ -1,4 +1,22 @@
-"""RethinkDB backend components ..."""
+"""RethinkDB backend implementation.
+
+Contains a RethinkDB-specific implementation of the
+:mod:`~bigchaindb.backend.changefeed`, :mod:`~bigchaindb.backend.query`, and
+:mod:`~bigchaindb.backend.schema` interfaces.
+
+You can specify BigchainDB to use RethinkDB as its database backend by either
+setting ``database.backend`` to ``'rethinkdb'`` in your configuration file, or
+setting the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable to
+``'rethinkdb'``.
+
+If configured to use RethinkDB, BigchainDB will automatically return instances
+of :class:`~bigchaindb.backend.rethinkdb.RethinkDBConnection` for
+:func:`~bigchaindb.backend.connection.connect` and dispatch calls of the
+generic backend interfaces to the implementations in this module.
+"""
 
 # Register the single dispatched modules on import.
 from bigchaindb.backend.rethinkdb import changefeed, schema, query  # noqa
+
+# RethinkDBConnection should always be accessed via
+# ``bigchaindb.backend.connect()``.

bigchaindb/backend/rethinkdb/connection.py

@@ -18,12 +18,12 @@ class RethinkDBConnection(Connection):
     """
 
     def __init__(self, host, port, dbname, max_tries=3):
-        """Create a new Connection instance.
+        """Create a new :class:`~.RethinkDBConnection` instance.
+
+        See :meth:`.Connection.__init__` for
+        :attr:`host`, :attr:`port`, and :attr:`dbname`.
 
         Args:
-            host (str): the host to connect to.
-            port (int): the port to connect to.
-            dbname (str): the name of the database to use.
             max_tries (int, optional): how many tries before giving up.
                 Defaults to 3.
         """

bigchaindb/backend/rethinkdb/query.py

@@ -1,5 +1,3 @@
-"""Query implementation for RethinkDB"""
-
 from time import time
 
 import rethinkdb as r

bigchaindb/backend/rethinkdb/schema.py

@@ -1,5 +1,3 @@
-"""Utils to initialize and drop the database."""
-
 import logging
 
 import rethinkdb as r

bigchaindb/backend/schema.py

@@ -1,4 +1,4 @@
-"""Schema-providing interfaces for backend databases"""
+"""Database creation and schema-providing interfaces for backends."""
 
 from functools import singledispatch
 

docs/server/source/appendices/backend.rst

@@ -1,41 +1,41 @@
-###############################################
+###########################
-:mod:`bigchaindb.backend` -- Backend Interfaces
+Database Backend Interfaces
-###############################################
+###########################
 
 .. automodule:: bigchaindb.backend
     :special-members: __init__
 
 
-Generic Backend
+Generic Interfaces
-===============
+==================
 
-:mod:`bigchaindb.backend.connection` -- Connection
+:mod:`bigchaindb.backend.connection`
---------------------------------------------------
+------------------------------------
 
 .. automodule:: bigchaindb.backend.connection
+    :special-members: __init__
 
-:mod:`bigchaindb.backend.schema` -- Schema
+:mod:`bigchaindb.backend.changefeed`
-------------------------------------------
+------------------------------------
-.. automodule:: bigchaindb.backend.schema
+.. automodule:: bigchaindb.backend.changefeed
 
-:mod:`bigchaindb.backend.query` -- Query
+:mod:`bigchaindb.backend.query`
-----------------------------------------
+-------------------------------
 .. automodule:: bigchaindb.backend.query
 
-:mod:`bigchaindb.backend.changefeed` -- Changefeed
+:mod:`bigchaindb.backend.schema`
---------------------------------------------------
+--------------------------------
-.. automodule:: bigchaindb.backend.changefeed
+.. automodule:: bigchaindb.backend.schema
 
 :mod:`bigchaindb.backend.utils`
 -------------------------------
 .. automodule:: bigchaindb.backend.utils
 
- 
+
-:mod:`bigchaindb.backend.rethinkdb` -- RethinkDB Backend
+RethinkDB Backend
-========================================================
+=================
 
 .. automodule:: bigchaindb.backend.rethinkdb
-    :special-members: __init__
 
 :mod:`bigchaindb.backend.rethinkdb.connection`
 ----------------------------------------------

tests/README.md

@@ -0,0 +1,18 @@
+# Tests
+
+## Test Structure
+
+Generally all tests are meant to be unit tests, with the exception of those in the [`integration/` folder](./integration/).
+
+A few notes:
+
+- [`common/`](./common/) contains self-contained tests only testing
+  [`bigchaindb/common/`](../bigchaindb/common/)
+- [`db/`](./db/) contains tests requiring the database backend (e.g. RethinkDB)
+
+## Pytest Customizations
+
+Customizations we've added to `pytest`:
+
+- `--database-backend`: Defines the backend to use for the tests. Must be one of the backends
+  available in the [server configuration](https://docs.bigchaindb.com/projects/server/en/latest/server-reference/configuration.html)

tests/pipelines/test_utils.py

@@ -25,7 +25,7 @@ def mock_changefeed_data():
 
 @pytest.fixture
 def mock_changefeed_bigchain(mock_changefeed_data):
-    connection = Connection()
+    connection = Connection(host=None, port=None, dbname=None)
     connection.run = Mock(return_value=mock_changefeed_data)
     return Bigchain(connection=connection)