diff --git a/.travis.yml b/.travis.yml index e13715a..76b4775 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,15 @@ env: matrix: fast_finish: true include: + - python: 3.9 + env: + - PLANETMINT_DATABASE_BACKEND=tarantool + - PLANETMINT_DATABASE_SSL= + - python: 3.9 + env: + - PLANETMINT_DATABASE_BACKEND=tarantool + - PLANETMINT_DATABASE_SSL= + - PLANETMINT_CI_ABCI=enable - python: 3.9 env: - PLANETMINT_DATABASE_BACKEND=localmongodb @@ -34,7 +43,8 @@ matrix: env: - PLANETMINT_DATABASE_BACKEND=localmongodb - PLANETMINT_DATABASE_SSL= - - PLANETMINT_CI_ABCI=enable + - PLANETMINT_CI_ABCI=enable + - python: 3.9 env: - PLANETMINT_ACCEPTANCE_TEST=enable diff --git a/Dockerfile-all-in-one b/Dockerfile-all-in-one index ebf9df8..62cb65a 100644 --- a/Dockerfile-all-in-one +++ b/Dockerfile-all-in-one @@ -34,7 +34,7 @@ RUN mkdir -p /data/db /data/configdb \ # Planetmint enviroment variables ENV PLANETMINT_DATABASE_PORT 27017 -ENV PLANETMINT_DATABASE_BACKEND localmongodb +ENV PLANETMINT_DATABASE_BACKEND tarantool ENV PLANETMINT_SERVER_BIND 0.0.0.0:9984 ENV PLANETMINT_WSSERVER_HOST 0.0.0.0 ENV PLANETMINT_WSSERVER_SCHEME ws diff --git a/docker-compose.yml b/docker-compose.yml index a72b8bf..5736e36 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,15 @@ services: - "27017:27017" command: mongod restart: always + + tarantool: + image: tarantool/tarantool:2.3.1 + ports: + - "5200:5200" + - "3301:3301" + command: tarantool + restart: always + planetmint: depends_on: - mongodb @@ -31,9 +40,9 @@ services: - ./pytest.ini:/usr/src/app/pytest.ini - ./tox.ini:/usr/src/app/tox.ini environment: - PLANETMINT_DATABASE_BACKEND: localmongodb - PLANETMINT_DATABASE_HOST: mongodb - PLANETMINT_DATABASE_PORT: 27017 + PLANETMINT_DATABASE_BACKEND: tarantool + PLANETMINT_DATABASE_HOST: tarantool + PLANETMINT_DATABASE_PORT: 3301 PLANETMINT_SERVER_BIND: 0.0.0.0:9984 PLANETMINT_WSSERVER_HOST: 0.0.0.0 PLANETMINT_WSSERVER_ADVERTISED_HOST: planetmint @@ -43,6 +52,7 @@ services: - "9984:9984" - "9985:9985" - "26658" + - "2222:2222" healthcheck: test: ["CMD", "bash", "-c", "curl http://planetmint:9984 && curl http://tendermint:26657/abci_query"] interval: 3s @@ -50,6 +60,7 @@ services: retries: 3 command: '.ci/entrypoint.sh' restart: always + tendermint: image: tendermint/tendermint:v0.31.5 # volumes: @@ -60,6 +71,7 @@ services: - "26657:26657" command: sh -c "tendermint init && tendermint node --consensus.create_empty_blocks=false --proxy_app=tcp://planetmint:26658" restart: always + bdb: image: busybox depends_on: @@ -93,7 +105,7 @@ services: context: . dockerfile: Dockerfile-dev args: - backend: localmongodb + backend: tarantool volumes: - .:/usr/src/app/ command: make -C docs/root html diff --git a/planetmint/__init__.py b/planetmint/__init__.py index a8e927d..a852ffc 100644 --- a/planetmint/__init__.py +++ b/planetmint/__init__.py @@ -6,6 +6,7 @@ import copy import logging import os +#import planetmint.debug from planetmint.log import DEFAULT_LOGGING_CONFIG as log_config from planetmint.lib import Planetmint # noqa @@ -23,45 +24,57 @@ from planetmint.core import App # noqa # I tried to configure _database_keys_map = { # TODO Check if it is working after removing 'name' field - 'tarantool_db': ('host', 'port'), + 'tarantool': ('host', 'port'), + 'localmongodb': ('host', 'port', 'name'), } -_base_database_tarantool_local_db = { # TODO Rewrite this configs for tarantool usage +_base_database_localmongodb = { 'host': 'localhost', + 'port': 27017, + 'name': 'planetmint', + 'replicaset': None, + 'login': None, + 'password': None, +} + +_database_localmongodb = { + 'backend': 'localmongodb', + 'connection_timeout': 5000, + 'max_tries': 3, + 'ssl': False, + 'ca_cert': None, + 'certfile': None, + 'keyfile': None, + 'keyfile_passphrase': None, + 'crlfile': None, +} +_database_localmongodb.update(_base_database_localmongodb) + +_base_database_tarantool_local_db = { # TODO Rewrite this configs for tarantool usage + 'host': 'tarantool', 'port': 3301, - 'username': None, + 'username': 'guest', 'password': None, "connect_now": True, "encoding": "utf-8" } -init_config = { - "init_file": "init_db.txt", - "relative_path": os.path.dirname(os.path.abspath(__file__)) + "/backend/tarantool/" -} -drop_config = { - "drop_file": "drop_db.txt", # planetmint/backend/tarantool/init_db.txt - "relative_path": os.path.dirname(os.path.abspath(__file__)) + "/backend/tarantool/" -} _database_tarantool = { - 'backend': 'tarantool_db', + 'backend': 'tarantool', 'connection_timeout': 5000, 'max_tries': 3, "reconnect_delay": 0.5, - "ctl_config": { - "login": "admin", - "host": "admin:pass@127.0.0.1:3301", - "service": "tarantoolctl connect", - "init_config": init_config, - "drop_config": drop_config - } + #"host": "admin:pass@127.0.0.1:3301", + #"service": "tarantoolctl connect", } _database_tarantool.update(_base_database_tarantool_local_db) _database_map = { - 'tarantool_db': _database_tarantool + 'tarantool': _database_tarantool, + 'localmongodb': _database_localmongodb, } + config = { 'server': { # Note: this section supports all the Gunicorn settings: @@ -85,7 +98,7 @@ config = { 'version': 'v0.31.5', # look for __tm_supported_versions__ }, # TODO Maybe remove hardcode configs for tarantool (review) - 'database': _database_map['tarantool_db'], + 'database': _database_map['tarantool'], 'log': { 'file': log_config['handlers']['file']['filename'], 'error_file': log_config['handlers']['errors']['filename'], diff --git a/planetmint/backend/connection_tarantool.py b/planetmint/backend/connection_tarantool.py index 1954f37..f3eb4c7 100644 --- a/planetmint/backend/connection_tarantool.py +++ b/planetmint/backend/connection_tarantool.py @@ -9,19 +9,30 @@ from itertools import repeat import tarantool import planetmint +import os from planetmint.backend.exceptions import ConnectionError from planetmint.backend.utils import get_planetmint_config_value, get_planetmint_config_value_or_key_error from planetmint.common.exceptions import ConfigurationError BACKENDS = { # This is path to MongoDBClass - 'tarantool_db': 'planetmint.backend.connection_tarantool.TarantoolDB', + 'tarantool': 'planetmint.backend.connection_tarantool.TarantoolDB', } logger = logging.getLogger(__name__) class TarantoolDB: + init_config = { + "init_file": "init_db.txt", + "relative_path": os.path.dirname(os.path.abspath(__file__)) + "/backend/tarantool/" + } + + drop_config = { + "drop_file": "drop_db.txt", # planetmint/backend/tarantool/init_db.txt + "relative_path": os.path.dirname(os.path.abspath(__file__)) + "/backend/tarantool/" + } + def __init__(self, host: str, port: int, user: str, password: str, reset_database: bool = False): self.db_connect = tarantool.connect(host=host, port=port, user=user, password=password) if reset_database: @@ -40,23 +51,25 @@ class TarantoolDB: def drop_database(self): from planetmint.backend.tarantool.utils import run - config = get_planetmint_config_value_or_key_error("ctl_config") - drop_config = config["drop_config"] - f_path = "%s%s" % (drop_config["relative_path"], drop_config["drop_file"]) +# config = get_planetmint_config_value_or_key_error("ctl_config") +# drop_config = config["drop_config"] + f_path = "%s%s" % (self.drop_config["relative_path"], self.drop_config["drop_file"]) commands = self.__read_commands(file_path=f_path) run(commands=commands, config=config) def init_database(self): from planetmint.backend.tarantool.utils import run - config = get_planetmint_config_value_or_key_error("ctl_config") - init_config = config["init_config"] - f_path = "%s%s" % (init_config["relative_path"], init_config["init_file"]) + # config = get_planetmint_config_value_or_key_error("ctl_config") + # init_config = config["init_config"] + f_path = "%s%s" % (self.init_config["relative_path"], self.init_config["init_file"]) commands = self.__read_commands(file_path=f_path) run(commands=commands, config=config) -def connect(host: str = None, port: int = None, username: str = "admin", password: str = "pass", + + +def connect(host: str = None, port: int = None, username: str = None, password: str = None, backend: str = None, reset_database: bool = False, name=None, max_tries=None, - connection_timeout=None, replicaset=None, ssl=None, login: str = "admin", ctl_config=None, + connection_timeout=None, replicaset=None, ssl=None, login: str = None, ctl_config=None, ca_cert=None, certfile=None, keyfile=None, keyfile_passphrase=None, reconnect_delay=None, crlfile=None, connect_now=True, encoding=None): backend = backend or get_planetmint_config_value_or_key_error('backend') # TODO Rewrite Configs @@ -73,7 +86,10 @@ def connect(host: str = None, port: int = None, username: str = "admin", passwor 'Planetmint currently supports {}'.format(backend, BACKENDS.keys())) except (ImportError, AttributeError) as exc: raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc - + print(host) + print(port) + print(username) + logger.debug('Connection: {}'.format(Class)) return Class(host=host, port=port, user=username, password=password, reset_database=reset_database) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 414c672..f6bc958 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -5,7 +5,7 @@ """Query implementation for MongoDB""" -from pymongo import DESCENDING +#from pymongo import DESCENDING from secrets import token_hex from operator import itemgetter diff --git a/planetmint/commands/planetmint.py b/planetmint/commands/planetmint.py index c44afa8..d03ee61 100644 --- a/planetmint/commands/planetmint.py +++ b/planetmint/commands/planetmint.py @@ -318,12 +318,11 @@ def create_parser(): help='Prepare the config file.') config_parser.add_argument('backend', - choices=['tarantool_db'], - default='tarantool_db', - const='tarantool_db', + choices=['tarantool', 'localmongodb'], + default='tarantool', + const='tarantool', nargs='?', - help='The backend to use. It can only be ' - '"tarantool_db", currently.') + help='The backend to use. Tarantool is default.') # parser for managing elections election_parser = subparsers.add_parser('election', diff --git a/setup.py b/setup.py index 9231afa..31198b6 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ install_requires = [ 'gunicorn==20.1.0', 'jsonschema==3.2.0', 'logstats==0.3.0', - 'packaging>=20.9', + 'packaging>=20.9', # TODO Consider not installing the db drivers, or putting them in extras. 'pymongo==3.11.4', 'tarantool==0.7.1', @@ -91,9 +91,10 @@ install_requires = [ 'pyyaml==5.4.1', 'requests==2.25.1', 'setproctitle==1.2.2', + 'ptvsd' ] -if sys.version_info < (3, 6): +if sys.version_info < (3, 9): install_requires.append('pysha3~=1.0.2') setup( diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index b6c90ad..0fe1b33 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -181,7 +181,7 @@ def test_run_configure_when_config_does_exist(monkeypatch, @pytest.mark.skip @pytest.mark.parametrize('backend', ( - 'localmongodb', + 'tarantool', )) def test_run_configure_with_backend(backend, monkeypatch, mock_write_config): import planetmint diff --git a/tests/conftest.py b/tests/conftest.py index 726c45b..6d2cd8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,7 +48,7 @@ def pytest_addoption(parser): parser.addoption( '--database-backend', action='store', - default=os.environ.get('PLANETMINT_DATABASE_BACKEND', 'tarantool_db'), + default=os.environ.get('PLANETMINT_DATABASE_BACKEND', 'tarantool'), help='Defines the backend to use (available: {})'.format(backends), ) @@ -97,7 +97,7 @@ def _configure_planetmint(request): test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) # backend = request.config.getoption('--database-backend') - backend = "tarantool_db" + backend = "tarantool" config = { 'database': planetmint._database_map[backend], diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 2b50135..c529dfb 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -185,7 +185,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): config_utils.autoconfigure() database_mongodb = { - 'backend': 'localmongodb', + 'backend': 'tarantool', 'host': DATABASE_HOST, 'port': DATABASE_PORT, 'name': DATABASE_NAME,