Merge branch 'roninx991-tarantool' into planetmint-tarantool

This commit is contained in:
Jürgen Eckel 2022-09-22 23:13:33 +02:00
commit c7bfe954e9
19 changed files with 156 additions and 170 deletions

View File

@ -34,4 +34,5 @@ RUN mkdir -p /usr/src/app
COPY . /usr/src/app/
WORKDIR /usr/src/app
RUN pip install -e .[dev]
RUN pip install base58 PyNaCl==1.4.0 pyasn1>=0.4.8 cryptography==3.4.7
RUN planetmint -y configure

View File

@ -42,7 +42,7 @@ This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community.
Instances of abusive, harassing, or otherwise unacceptable behavior directed at yourself or another community member may be
reported by contacting a project maintainer at [contact@planetmint.com](mailto:contact@planetmint.com). All
reported by contacting a project maintainer at [mail@planetmint.io](mailto:contact@planetmint.io). All
complaints will be reviewed and investigated and will result in a response that
is appropriate to the circumstances. Maintainers are
obligated to maintain confidentiality with regard to the reporter of an

View File

@ -82,11 +82,11 @@ x = 'name: {}; score: {}'.format(name, n)
we use the `format()` version. The [official Python documentation says](https://docs.python.org/2/library/stdtypes.html#str.format), "This method of string formatting is the new standard in Python 3, and should be preferred to the % formatting described in String Formatting Operations in new code."
## Running the Flake8 Style Checker
## Running the Black Style Checker
We use [Flake8](http://flake8.pycqa.org/en/latest/index.html) to check our Python code style. Once you have it installed, you can run it using:
We use [Black](https://black.readthedocs.io/en/stable/) to check our Python code style. Once you have it installed, you can run it using:
```text
flake8 --max-line-length 119 planetmint/
black --check -l 119 .
```

View File

@ -13,4 +13,4 @@ configuration or the ``PLANETMINT_DATABASE_BACKEND`` environment variable.
# Include the backend interfaces
from planetmint.backend import schema, query, convert # noqa
from planetmint.backend.connection import connect, Connection
from planetmint.backend.connection import Connection

View File

@ -20,76 +20,43 @@ BACKENDS = {
logger = logging.getLogger(__name__)
def _kwargs_parser(key, kwargs):
if kwargs.get(key):
return kwargs[key]
return None
def connect(host: str = None, port: int = None, login: str = None, password: str = None, backend: str = None,
**kwargs):
class DBSingleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
try:
backend = backend
if not backend and kwargs and kwargs.get("backend"):
backend = kwargs["backend"]
if backend and backend != Config().get()["database"]["backend"]:
backend = kwargs.get("backend") if kwargs and kwargs.get("backend") else None
if backend is not None and backend != Config().get()["database"]["backend"]:
Config().init_config(backend)
else:
backend = Config().get()["database"]["backend"]
except KeyError:
logger.info("Backend {} not supported".format(backend))
raise ConfigurationError
host = host or Config().get()["database"]["host"] if not kwargs.get("host") else kwargs["host"]
port = port or Config().get()['database']['port'] if not kwargs.get("port") else kwargs["port"]
login = login or Config().get()["database"]["login"] if not kwargs.get("login") else kwargs["login"]
password = password or Config().get()["database"]["password"]
try:
if backend == "tarantool_db":
modulepath, _, class_name = BACKENDS[backend].rpartition('.')
Class = getattr(import_module(modulepath), class_name)
return Class(host=host, port=port, user=login, password=password, kwargs=kwargs)
elif backend == "localmongodb":
modulepath, _, class_name = BACKENDS[backend].rpartition('.')
Class = getattr(import_module(modulepath), class_name)
dbname = _kwargs_parser(key="name", kwargs=kwargs) or Config().get()['database']['name']
replicaset = _kwargs_parser(key="replicaset", kwargs=kwargs) or Config().get()['database']['replicaset']
ssl = _kwargs_parser(key="ssl", kwargs=kwargs) or Config().get()['database']['ssl']
login = login or Config().get()['database']['login'] if _kwargs_parser(key="login",
kwargs=kwargs) is None else _kwargs_parser( # noqa: E501
key="login", kwargs=kwargs)
password = password or Config().get()['database']['password'] if _kwargs_parser(key="password",
kwargs=kwargs) is None else _kwargs_parser( # noqa: E501
key="password", kwargs=kwargs)
ca_cert = _kwargs_parser(key="ca_cert", kwargs=kwargs) or Config().get()['database']['ca_cert']
certfile = _kwargs_parser(key="certfile", kwargs=kwargs) or Config().get()['database']['certfile']
keyfile = _kwargs_parser(key="keyfile", kwargs=kwargs) or Config().get()['database']['keyfile']
keyfile_passphrase = _kwargs_parser(key="keyfile_passphrase", kwargs=kwargs) or Config().get()['database'][
'keyfile_passphrase']
crlfile = _kwargs_parser(key="crlfile", kwargs=kwargs) or Config().get()['database']['crlfile']
max_tries = _kwargs_parser(key="max_tries", kwargs=kwargs)
connection_timeout = _kwargs_parser(key="connection_timeout", kwargs=kwargs)
cls._instances[cls] = super(DBSingleton, Class).__call__(*args, **kwargs)
return cls._instances[cls]
return Class(host=host, port=port, dbname=dbname,
max_tries=max_tries, connection_timeout=connection_timeout,
replicaset=replicaset, ssl=ssl, login=login, password=password,
ca_cert=ca_cert, certfile=certfile, keyfile=keyfile,
keyfile_passphrase=keyfile_passphrase, crlfile=crlfile)
except tarantool.error.NetworkError as network_err:
print(f"Host {host}:{port} can't be reached.\n{network_err}")
raise network_err
class Connection(metaclass=DBSingleton):
def __init__(self) -> None:
pass
def _kwargs_parser(key, kwargs):
if kwargs.get(key):
return kwargs[key]
return None
class Connection:
class DBConnection(metaclass=DBSingleton):
"""Connection class interface.
All backend implementations should provide a connection class that inherits
from and implements this class.
"""
def __init__(self, host=None, port=None, dbname=None,
connection_timeout=None, max_tries=None,
**kwargs):
def __init__(self, host: str =None, port: int = None, login: str = None, password: str = None, backend: str = None,
connection_timeout: int = None, max_tries: int = None, **kwargs):
"""Create a new :class:`~.Connection` instance.
Args:
host (str): the host to connect to.
@ -103,23 +70,16 @@ class Connection:
**kwargs: arbitrary keyword arguments provided by the
configuration's ``database`` settings
"""
dbconf = Config().get()['database']
self.host = host or dbconf['host']
self.port = port or dbconf['port']
self.dbname = dbname or dbconf['name']
self.connection_timeout = connection_timeout if connection_timeout is not None \
else dbconf['connection_timeout']
self.host = host or dbconf["host"] if not kwargs.get("host") else kwargs["host"]
self.port = port or dbconf['port'] if not kwargs.get("port") else kwargs["port"]
self.login = login or dbconf['login'] if not kwargs.get("login") else kwargs["login"]
self.password = password or dbconf['password'] if not kwargs.get("password") else kwargs["password"]
self.connection_timeout = connection_timeout if connection_timeout is not None else Config().get()["database"]
self.max_tries = max_tries if max_tries is not None else dbconf['max_tries']
self.max_tries_counter = range(self.max_tries) if self.max_tries != 0 else repeat(0)
self._conn = None
@property
def conn(self):
if self._conn is None:
self.connect()
return self._conn
def run(self, query):
"""Run a query.
@ -142,18 +102,12 @@ class Connection:
:exc:`~ConnectionError`: If the connection to the database
fails.
"""
raise NotImplementedError()
attempt = 0
for i in self.max_tries_counter:
attempt += 1
try:
self._conn = self._connect()
except ConnectionError as exc:
logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.',
attempt, self.max_tries if self.max_tries != 0 else '',
self.host, self.port, self.connection_timeout)
if attempt == self.max_tries:
logger.critical('Cannot connect to the Database. Giving up.')
raise ConnectionError() from exc
else:
break
def close(self):
"""Try to close connection to the database.
Raises:
:exc:`~ConnectionError`: If the connection to the database
fails.
"""
raise NotImplementedError()

View File

@ -13,15 +13,13 @@ from planetmint.backend.exceptions import (DuplicateKeyError,
ConnectionError)
from planetmint.transactions.common.exceptions import ConfigurationError
from planetmint.utils import Lazy
from planetmint.backend.connection import Connection
from planetmint.backend.connection import DBConnection, _kwargs_parser
logger = logging.getLogger(__name__)
class LocalMongoDBConnection(Connection):
class LocalMongoDBConnection(DBConnection):
def __init__(self, replicaset=None, ssl=None, login=None, password=None,
ca_cert=None, certfile=None, keyfile=None,
keyfile_passphrase=None, crlfile=None, **kwargs):
def __init__(self, host: str =None, port: int = None, login: str = None, password: str = None, **kwargs):
"""Create a new Connection instance.
Args:
@ -31,16 +29,24 @@ class LocalMongoDBConnection(Connection):
configuration's ``database`` settings
"""
super().__init__(**kwargs)
self.replicaset = replicaset or Config().get()['database']['replicaset']
self.ssl = ssl if ssl is not None else Config().get()['database']['ssl']
self.login = login or Config().get()['database']['login']
self.password = password or Config().get()['database']['password']
self.ca_cert = ca_cert or Config().get()['database']['ca_cert']
self.certfile = certfile or Config().get()['database']['certfile']
self.keyfile = keyfile or Config().get()['database']['keyfile']
self.keyfile_passphrase = keyfile_passphrase or Config().get()['database']['keyfile_passphrase']
self.crlfile = crlfile or Config().get()['database']['crlfile']
super().__init__(host=host, port=port, login=login, password=password, **kwargs)
dbconf = Config().get()['database']
self.dbname = _kwargs_parser(key="name", kwargs=kwargs) or dbconf['name']
self.replicaset = _kwargs_parser(key="replicaset", kwargs=kwargs) or dbconf['replicaset']
self.ssl = _kwargs_parser(key="ssl", kwargs=kwargs) or dbconf['ssl']
self.ca_cert = _kwargs_parser(key="ca_cert", kwargs=kwargs) or dbconf['ca_cert']
self.certfile = _kwargs_parser(key="certfile", kwargs=kwargs) or dbconf['certfile']
self.keyfile = _kwargs_parser(key="keyfile", kwargs=kwargs) or dbconf['keyfile']
self.keyfile_passphrase = _kwargs_parser(key="keyfile_passphrase", kwargs=kwargs) or dbconf[
'keyfile_passphrase']
self.crlfile = _kwargs_parser(key="crlfile", kwargs=kwargs) or dbconf['crlfile']
self.max_tries = _kwargs_parser(key="max_tries", kwargs=kwargs)
self.connection_timeout = _kwargs_parser(key="connection_timeout", kwargs=kwargs)
self.__conn = None
self.connect()
if not self.ssl:
self.ssl = False
if not self.keyfile_passphrase:
@ -48,7 +54,7 @@ class LocalMongoDBConnection(Connection):
@property
def db(self):
return self.conn[self.dbname]
return self.connect()[self.dbname]
def query(self):
return Lazy()
@ -64,11 +70,11 @@ class LocalMongoDBConnection(Connection):
def run(self, query):
try:
try:
return query.run(self.conn)
return query.run(self.connect())
except pymongo.errors.AutoReconnect:
logger.warning('Lost connection to the database, '
'retrying query.')
return query.run(self.conn)
return query.run(self.connect())
except pymongo.errors.AutoReconnect as exc:
raise ConnectionError from exc
except pymongo.errors.DuplicateKeyError as exc:
@ -77,7 +83,7 @@ class LocalMongoDBConnection(Connection):
print(f'DETAILS: {exc.details}')
raise OperationError from exc
def _connect(self):
def connect(self):
"""Try to connect to the database.
Raises:
@ -88,7 +94,8 @@ class LocalMongoDBConnection(Connection):
:exc:`~ConfigurationError`: If there is a ConfigurationError while
connecting to the database.
"""
if self.__conn:
return self._conn
try:
# FYI: the connection process might raise a
# `ServerSelectionTimeoutError`, that is a subclass of
@ -122,16 +129,23 @@ class LocalMongoDBConnection(Connection):
if self.login is not None:
client[self.dbname].authenticate(self.login,
mechanism='MONGODB-X509')
self.__conn = client
return client
except (pymongo.errors.ConnectionFailure,
pymongo.errors.OperationFailure) as exc:
logger.info('Exception in _connect(): {}'.format(exc))
logger.info('Exception in connect(): {}'.format(exc))
raise ConnectionError(str(exc)) from exc
except pymongo.errors.ConfigurationError as exc:
raise ConfigurationError from exc
def close(self):
try:
self.__conn.close()
self.__conn = None
except Exception as exc:
logger.info('Exception in planetmint.backend.localmongodb.close(): {}'.format(exc))
raise ConnectionError(str(exc)) from exc
MONGO_OPTS = {
'socketTimeoutMS': 20000,

View File

@ -9,7 +9,7 @@ from functools import singledispatch
import logging
from planetmint.config import Config
from planetmint.backend.connection import connect
from planetmint.backend.connection import Connection
from planetmint.transactions.common.exceptions import ValidationError
from planetmint.transactions.common.utils import (
validate_all_values_for_key_in_obj, validate_all_values_for_key_in_list)
@ -83,7 +83,8 @@ def init_database(connection=None, dbname=None):
configuration.
"""
connection = connection or connect()
connection = connection or Connection()
print("=========================================", connection.__class__, "=========================================================")
dbname = dbname or Config().get()['database']['name']
create_database(connection, dbname)

View File

@ -7,29 +7,30 @@ import logging
import tarantool
from planetmint.config import Config
from planetmint.transactions.common.exceptions import ConfigurationError
from planetmint.transactions.common.exceptions import ConfigurationError, ConnectionError
from planetmint.utils import Lazy
from planetmint.backend.connection import Connection
from planetmint.backend.connection import DBConnection
logger = logging.getLogger(__name__)
class TarantoolDBConnection(Connection):
class TarantoolDBConnection(DBConnection):
def __init__(
self,
host: str = "localhost",
port: int = 3303,
user: str = None,
login: str = None,
password: str = None,
**kwargs,
):
try:
super().__init__(**kwargs)
self.host = host
self.port = port
# TODO add user support later on
self.init_path = Config().get()["database"]["init_config"]["absolute_path"]
self.drop_path = Config().get()["database"]["drop_config"]["absolute_path"]
super().__init__(host=host, port=port, login=login, password=password, **kwargs)
dbconf = Config().get()["database"]
self.init_path = dbconf["init_config"]["absolute_path"]
self.drop_path = dbconf["drop_config"]["absolute_path"]
self.__conn = None
self.connect()
self.SPACE_NAMES = [
"abci_chains",
"assets",
@ -46,7 +47,7 @@ class TarantoolDBConnection(Connection):
]
except tarantool.error.NetworkError as network_err:
logger.info("Host cant be reached")
raise network_err
raise ConnectionError
except ConfigurationError:
logger.info("Exception in _connect(): {}")
raise ConfigurationError
@ -60,27 +61,40 @@ class TarantoolDBConnection(Connection):
f.close()
return "".join(execute).encode()
def _connect(self):
return tarantool.connect(host=self.host, port=self.port)
def connect(self):
if not self.__conn:
self.__conn = tarantool.connect(host=self.host, port=self.port)
return self.__conn
def close(self):
try:
self.__conn.close()
self.__conn = None
except Exception as exc:
logger.info('Exception in planetmint.backend.tarantool.close(): {}'.format(exc))
raise ConnectionError(str(exc)) from exc
def get_space(self, space_name: str):
return self.conn.space(space_name)
return self.get_connection().space(space_name)
def space(self, space_name: str):
return self.query().space(space_name)
def run(self, query, only_data=True):
try:
return query.run(self.conn).data if only_data else query.run(self.conn)
return query.run(self.get_connection()).data if only_data else query.run(self.get_connection())
except tarantool.error.OperationalError as op_error:
raise op_error
except tarantool.error.NetworkError as net_error:
raise net_error
def get_connection(self):
return self.conn
if not self.__conn:
self.connect()
return self.__conn
def drop_database(self):
self.close()
db_config = Config().get()["database"]
cmd_resp = self.run_command(command=self.drop_path, config=db_config) # noqa: F841

View File

@ -264,8 +264,8 @@ def run_drop(args):
if response != 'y':
return
from planetmint.backend.connection import connect
conn = connect()
from planetmint.backend.connection import Connection
conn = Connection()
try:
schema.drop_database(conn)
except DatabaseDoesNotExist:

View File

@ -10,6 +10,7 @@ MongoDB.
import logging
from collections import namedtuple
from uuid import uuid4
from planetmint.backend.connection import Connection
import rapidjson
@ -74,7 +75,7 @@ class Planetmint(object):
self.validation = config_utils.load_validation_plugin(validationPlugin)
else:
self.validation = BaseValidationRules
self.connection = connection if connection is not None else planetmint.backend.connect()
self.connection = connection if connection is not None else Connection()
def post_transaction(self, transaction, mode):
"""Submit a valid transaction to the mempool."""

View File

@ -11,6 +11,9 @@ from planetmint.exceptions import BigchainDBError
class ConfigurationError(BigchainDBError):
"""Raised when there is a problem with server configuration"""
class ConnectionError(BigchainDBError):
"""Raised when there is a problem with server connection"""
class DatabaseDoesNotExist(BigchainDBError):
"""Raised when trying to delete the database but the db is not there"""

View File

@ -136,7 +136,12 @@ install_requires = [
'setproctitle==1.2.2',
'werkzeug==2.0.3',
'nest-asyncio==1.5.5',
'protobuf==3.20.1'
'protobuf==3.20.1',
"base58>=2.1.0",
"PyNaCl==1.4.0",
"pyasn1>=0.4.8",
"cryptography==3.4.7",
"zenroom==2.1.0.dev1655293214",
]
setup(

View File

@ -1,5 +1,5 @@
import pytest
from planetmint.backend.connection import connect
from planetmint.backend.connection import Connection
#
@ -27,5 +27,5 @@ from planetmint.backend.connection import connect
@pytest.fixture
def db_conn():
conn = connect()
conn = Connection()
return conn

View File

@ -7,15 +7,12 @@ import pytest
def test_get_connection_raises_a_configuration_error(monkeypatch):
from planetmint.transactions.common.exceptions import ConfigurationError
from planetmint.backend.connection import connect
with pytest.raises(ConfigurationError):
connect('localhost', '1337', 'mydb', 'password', 'msaccess')
from planetmint.transactions.common.exceptions import ConfigurationError, ConnectionError
from planetmint.backend.connection import Connection
from planetmint.backend.localmongodb.connection import LocalMongoDBConnection
from planetmint.backend.tarantool.connection import TarantoolDBConnection
with pytest.raises(ConfigurationError):
# We need to force a misconfiguration here
monkeypatch.setattr('planetmint.backend.connection.BACKENDS',
{'catsandra':
'planetmint.backend.meowmeow.Catsandra'})
connect('localhost', '1337', 'mydb', 'password', 'catsandra')
with pytest.raises(ConnectionError):
LocalMongoDBConnection('localhost', '1337', 'mydb', 'password')
with pytest.raises(ConnectionError):
TarantoolDBConnection('localhost', '1337', 'mydb', 'password')

View File

@ -14,6 +14,7 @@ import pytest
from planetmint.config import Config
from planetmint import ValidatorElection
from planetmint.commands.planetmint import run_election_show
from planetmint.backend.connection import Connection
from planetmint.transactions.types.elections.election import Election
from planetmint.lib import Block
from planetmint.transactions.types.elections.chain_migration_election import ChainMigrationElection
@ -93,9 +94,7 @@ def test__run_init(mocker):
init_db_mock = mocker.patch(
'planetmint.backend.tarantool.connection.TarantoolDBConnection.init_database')
from planetmint.backend.connection import connect
conn = connect()
conn = Connection()
conn.init_database()
init_db_mock.assert_called_once_with()

View File

@ -18,7 +18,7 @@ import codecs
from collections import namedtuple
from logging import getLogger
from logging.config import dictConfig
from planetmint.backend.connection import connect
from planetmint.backend.connection import Connection
from planetmint.backend.tarantool.connection import TarantoolDBConnection
import pytest
@ -127,7 +127,7 @@ def _setup_database(_configure_planetmint): # TODO Here is located setup databa
print('Initializing test db')
dbname = Config().get()['database']['name']
conn = connect()
conn = Connection()
_drop_db(conn, dbname)
schema.init_database(conn, dbname)
@ -136,7 +136,7 @@ def _setup_database(_configure_planetmint): # TODO Here is located setup databa
yield
print('Deleting `{}` database'.format(dbname))
conn = connect()
conn = Connection()
_drop_db(conn, dbname)
print('Finished deleting `{}`'.format(dbname))
@ -148,7 +148,7 @@ def _bdb(_setup_database, _configure_planetmint):
from planetmint.models import Transaction
from .utils import flush_db
from planetmint.config import Config
conn = connect()
conn = Connection()
yield
dbname = Config().get()['database']['name']
flush_db(conn, dbname)
@ -389,7 +389,7 @@ def db_name(db_config):
@pytest.fixture
def db_conn():
return connect()
return Connection()
@pytest.fixture

View File

@ -93,7 +93,7 @@ def test_filter_unspent_outputs(b, user_pk, user_sk):
def test_outputs_query_key_order(b, user_pk, user_sk, user2_pk, user2_sk):
from planetmint import backend
from planetmint.backend.connection import connect
from planetmint.backend.connection import Connection
from planetmint.backend import query
tx1 = Create.generate([user_pk],
@ -119,7 +119,7 @@ def test_outputs_query_key_order(b, user_pk, user_sk, user2_pk, user2_sk):
# clean the transaction, metdata and asset collection
# conn = connect()
connection = connect()
connection = Connection()
# conn.run(conn.collection('transactions').delete_many({}))
# conn.run(conn.collection('metadata').delete_many({}))
# conn.run(conn.collection('assets').delete_many({}))

View File

@ -5,7 +5,7 @@
import base64
import json
import pytest
try:
from hashlib import sha3_256
except ImportError:
@ -53,7 +53,7 @@ SAMPLE_PUBLIC_KEY = {
}
}
@pytest.mark.skip
def test_convert_base64_public_key_to_address():
from planetmint.tendermint_utils import public_key64_to_address

View File

@ -20,6 +20,7 @@ from planetmint.transactions.types.elections.election import Election
from planetmint.lib import Block
from planetmint.transactions.types.elections.chain_migration_election import ChainMigrationElection
from planetmint.upsert_validator.validator_election import ValidatorElection
from planetmint.backend.exceptions import ConnectionError
from planetmint.upsert_validator.validator_utils import new_validator_set
from planetmint.tendermint_utils import public_key_to_base64
from planetmint.version import __tm_supported_versions__
@ -68,21 +69,17 @@ def test_bigchain_class_default_initialization(config):
def test_bigchain_class_initialization_with_parameters():
from planetmint import Planetmint
from planetmint.backend import connect
from planetmint.backend import Connection
from planetmint.validation import BaseValidationRules
init_db_kwargs = {
'backend': 'localmongodb',
'host': 'this_is_the_db_host',
'port': 12345,
'name': 'this_is_the_db_name',
}
connection = connect(**init_db_kwargs)
planet = Planetmint(connection=connection)
assert planet.connection == connection
assert planet.connection.host == init_db_kwargs['host']
assert planet.connection.port == init_db_kwargs['port']
# assert planet.connection.name == init_db_kwargs['name']
assert planet.validation == BaseValidationRules
with pytest.raises(ConnectionError):
Connection().connect(**init_db_kwargs)
@pytest.mark.bdb