From 8a68e24e695d68dbc149e20ad53003b74c963ca7 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 12 Dec 2016 18:28:43 +0100 Subject: [PATCH] Abstract db layer cherrypick docs (#932) * Add README.md to tests/ to describe test structure and pytest customizations * Add package-level docstrings to backends * Add README.md explaining the backend architecture * Small improvements to docstrings in backend modules * Restructure the backend automodule docs * Add more docstrings to backend connections * Add init to base backend Connection class to document expected interface * Reword the backend/README.md docs --- bigchaindb/backend/README.md | 45 ++++++++++++++++++++++ bigchaindb/backend/__init__.py | 8 +++- bigchaindb/backend/changefeed.py | 2 +- bigchaindb/backend/connection.py | 34 +++++++++++++++- bigchaindb/backend/query.py | 2 +- bigchaindb/backend/rethinkdb/__init__.py | 20 +++++++++- bigchaindb/backend/rethinkdb/connection.py | 8 ++-- bigchaindb/backend/rethinkdb/query.py | 2 - bigchaindb/backend/rethinkdb/schema.py | 2 - bigchaindb/backend/schema.py | 2 +- docs/server/source/appendices/backend.rst | 38 +++++++++--------- tests/README.md | 18 +++++++++ tests/pipelines/test_utils.py | 2 +- 13 files changed, 148 insertions(+), 35 deletions(-) create mode 100644 bigchaindb/backend/README.md create mode 100644 tests/README.md diff --git a/bigchaindb/backend/README.md b/bigchaindb/backend/README.md new file mode 100644 index 00000000..44217c34 --- /dev/null +++ b/bigchaindb/backend/README.md @@ -0,0 +1,45 @@ +# Backend Interfaces + +## Structure + +- [`changefeed.py`](./changefeed.py): Changefeed-related interfaces +- [`connection.py`](./connection.py): Database connection-related interfaces +- [`query.py`](./query.py): Database query-related interfaces, dispatched through single-dispatch +- [`schema.py`](./schema.py): Database setup and schema-related interfaces, dispatched through + single-dispatch + +Built-in implementations (e.g. [RethinkDB's](./rethinkdb)) are provided in sub-directories and +have their connection type's location exposed as `BACKENDS` in [`connection.py`](./connection.py). + +## Single-Dispatched Interfaces + +The architecture of this module is based heavily upon Python's newly-introduced [single-dispatch +generic functions](https://www.python.org/dev/peps/pep-0443/). Single-dispatch is convenient, +because it allows Python, rather than something we design ourselves, to manage the dispatching of +generic functions based on certain conditions being met (e.g. the database backend to use). + +To see what this looks like in BigchainDB, first note that our backend interfaces have been +configured to dispatch based on a backend's **connection type**. + +Call `bigchaindb.backend.connect()` to create an instance of a `Connection`: + +```python +from bigchaindb.backend import connect +connection = connect() # By default, uses the current configuration for backend, host, port, etc. +``` + +Then, we can call a backend function by directly calling its interface: + +```python +from bigchaindb.backend import query +query.write_transaction(connection, ...) +``` + +Notice that we don't need to care about which backend implementation to use or how to access it. +Code can simply call the base interface function with a `Connection` instance, and single-dispatch +will handle routing the call to the actual implementation. + +BigchainDB will load and register the configured backend's implementation automatically (see +`bigchaindb.backend.connect()`), so you should always just be able to call an interface function if +you have a `Connection` instance. A few helper utilities (see [`backend/utils.py`](./utils.py)) are +also provided to make registering new backend implementations easier. diff --git a/bigchaindb/backend/__init__.py b/bigchaindb/backend/__init__.py index 6f7c9f88..7e37b5d2 100644 --- a/bigchaindb/backend/__init__.py +++ b/bigchaindb/backend/__init__.py @@ -1,4 +1,10 @@ -"""Backend interfaces ...""" +"""Generic backend database interfaces expected by BigchainDB. + +The interfaces in this module allow BigchainDB to be agnostic about its +database backend. One can configure BigchainDB to use different databases as +its data store by setting the ``database.backend`` property in the +configuration or the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable. +""" # Include the backend interfaces from bigchaindb.backend import changefeed, schema, query # noqa diff --git a/bigchaindb/backend/changefeed.py b/bigchaindb/backend/changefeed.py index f42e5f71..ccba02d3 100644 --- a/bigchaindb/backend/changefeed.py +++ b/bigchaindb/backend/changefeed.py @@ -1 +1 @@ -"""Changefeed interfaces for backend databases.""" +"""Changefeed interfaces for backends.""" diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 0f0b9243..eea912bf 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -9,7 +9,10 @@ BACKENDS = { def connect(backend=None, host=None, port=None, name=None): - """Create a connection to the database backend. + """Create a new connection to the database backend. + + All arguments default to the current configuration's values if not + given. Args: backend (str): the name of the backend to use. @@ -18,7 +21,12 @@ def connect(backend=None, host=None, port=None, name=None): name (str): the name of the database to use. Returns: - An instance of :class:`~bigchaindb.backend.connection.Connection`. + An instance of :class:`~bigchaindb.backend.connection.Connection` + based on the given (or defaulted) :attr:`backend`. + + Raises: + :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend` + is not supported or could not be loaded. """ backend = backend or bigchaindb.config['database']['backend'] @@ -39,6 +47,28 @@ def connect(backend=None, host=None, port=None, name=None): class Connection: + """Connection class interface. + + All backend implementations should provide a connection class that + from and implements this class. + """ + + def __init__(self, host=None, port=None, dbname=None, *args, **kwargs): + """Create a new :class:`~.Connection` instance. + + Args: + host (str): the host to connect to. + port (int): the port to connect to. + dbname (str): the name of the database to use. + **kwargs: arbitrary keyword arguments provided by the + configuration's ``database`` settings + """ def run(self, query): + """Run a query. + + Args: + query: the query to run + """ + raise NotImplementedError() diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index e9fe0b46..88f215c4 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -1,4 +1,4 @@ -"""Query interfaces for backend databases.""" +"""Query interfaces for backends.""" from functools import singledispatch diff --git a/bigchaindb/backend/rethinkdb/__init__.py b/bigchaindb/backend/rethinkdb/__init__.py index 42b79735..22eaae5f 100644 --- a/bigchaindb/backend/rethinkdb/__init__.py +++ b/bigchaindb/backend/rethinkdb/__init__.py @@ -1,4 +1,22 @@ -"""RethinkDB backend components ...""" +"""RethinkDB backend implementation. + +Contains a RethinkDB-specific implementation of the +:mod:`~bigchaindb.backend.changefeed`, :mod:`~bigchaindb.backend.query`, and +:mod:`~bigchaindb.backend.schema` interfaces. + +You can specify BigchainDB to use RethinkDB as its database backend by either +setting ``database.backend`` to ``'rethinkdb'`` in your configuration file, or +setting the ``BIGCHAINDB_DATABASE_BACKEND`` environment variable to +``'rethinkdb'``. + +If configured to use RethinkDB, BigchainDB will automatically return instances +of :class:`~bigchaindb.backend.rethinkdb.RethinkDBConnection` for +:func:`~bigchaindb.backend.connection.connect` and dispatch calls of the +generic backend interfaces to the implementations in this module. +""" # Register the single dispatched modules on import. from bigchaindb.backend.rethinkdb import changefeed, schema, query # noqa + +# RethinkDBConnection should always be accessed via +# ``bigchaindb.backend.connect()``. diff --git a/bigchaindb/backend/rethinkdb/connection.py b/bigchaindb/backend/rethinkdb/connection.py index 03c2508c..173cdc7b 100644 --- a/bigchaindb/backend/rethinkdb/connection.py +++ b/bigchaindb/backend/rethinkdb/connection.py @@ -18,12 +18,12 @@ class RethinkDBConnection(Connection): """ def __init__(self, host, port, dbname, max_tries=3): - """Create a new Connection instance. + """Create a new :class:`~.RethinkDBConnection` instance. + + See :meth:`.Connection.__init__` for + :attr:`host`, :attr:`port`, and :attr:`dbname`. Args: - host (str): the host to connect to. - port (int): the port to connect to. - dbname (str): the name of the database to use. max_tries (int, optional): how many tries before giving up. Defaults to 3. """ diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index dda6aaf9..8fa6a512 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -1,5 +1,3 @@ -"""Query implementation for RethinkDB""" - from time import time import rethinkdb as r diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index 49a1a8e8..66ed7b20 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -1,5 +1,3 @@ -"""Utils to initialize and drop the database.""" - import logging import rethinkdb as r diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index b3ae8fab..b731a55f 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -1,4 +1,4 @@ -"""Schema-providing interfaces for backend databases""" +"""Database creation and schema-providing interfaces for backends.""" from functools import singledispatch diff --git a/docs/server/source/appendices/backend.rst b/docs/server/source/appendices/backend.rst index 2cb035e8..efad4afc 100644 --- a/docs/server/source/appendices/backend.rst +++ b/docs/server/source/appendices/backend.rst @@ -1,41 +1,41 @@ -############################################### -:mod:`bigchaindb.backend` -- Backend Interfaces -############################################### +########################### +Database Backend Interfaces +########################### .. automodule:: bigchaindb.backend :special-members: __init__ -Generic Backend -=============== +Generic Interfaces +================== -:mod:`bigchaindb.backend.connection` -- Connection --------------------------------------------------- +:mod:`bigchaindb.backend.connection` +------------------------------------ .. automodule:: bigchaindb.backend.connection + :special-members: __init__ -:mod:`bigchaindb.backend.schema` -- Schema ------------------------------------------- -.. automodule:: bigchaindb.backend.schema +:mod:`bigchaindb.backend.changefeed` +------------------------------------ +.. automodule:: bigchaindb.backend.changefeed -:mod:`bigchaindb.backend.query` -- Query ----------------------------------------- +:mod:`bigchaindb.backend.query` +------------------------------- .. automodule:: bigchaindb.backend.query -:mod:`bigchaindb.backend.changefeed` -- Changefeed --------------------------------------------------- -.. automodule:: bigchaindb.backend.changefeed +:mod:`bigchaindb.backend.schema` +-------------------------------- +.. automodule:: bigchaindb.backend.schema :mod:`bigchaindb.backend.utils` ------------------------------- .. automodule:: bigchaindb.backend.utils - -:mod:`bigchaindb.backend.rethinkdb` -- RethinkDB Backend -======================================================== + +RethinkDB Backend +================= .. automodule:: bigchaindb.backend.rethinkdb - :special-members: __init__ :mod:`bigchaindb.backend.rethinkdb.connection` ---------------------------------------------- diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..8e66a708 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,18 @@ +# Tests + +## Test Structure + +Generally all tests are meant to be unit tests, with the exception of those in the [`integration/` folder](./integration/). + +A few notes: + +- [`common/`](./common/) contains self-contained tests only testing + [`bigchaindb/common/`](../bigchaindb/common/) +- [`db/`](./db/) contains tests requiring the database backend (e.g. RethinkDB) + +## Pytest Customizations + +Customizations we've added to `pytest`: + +- `--database-backend`: Defines the backend to use for the tests. Must be one of the backends + available in the [server configuration](https://docs.bigchaindb.com/projects/server/en/latest/server-reference/configuration.html) diff --git a/tests/pipelines/test_utils.py b/tests/pipelines/test_utils.py index 1271068a..ebe0db75 100644 --- a/tests/pipelines/test_utils.py +++ b/tests/pipelines/test_utils.py @@ -25,7 +25,7 @@ def mock_changefeed_data(): @pytest.fixture def mock_changefeed_bigchain(mock_changefeed_data): - connection = Connection() + connection = Connection(host=None, port=None, dbname=None) connection.run = Mock(return_value=mock_changefeed_data) return Bigchain(connection=connection)