added some basic fixes

Signed-off-by: Jürgen Eckel <juergen@riddleandcode.com>

docker-compose.yml

@@ -15,16 +15,21 @@ services:
     command: mongod
     restart: always
   tarantool:
-    image: tarantool/tarantool:2.3.1
+    image: tarantool/tarantool:2.8.3
     ports:
       - "5200:5200"
       - "3301:3301"
-    command: tarantool
+      - "3303:3303"
+      - "8081:8081"
+    volumes:
+      - ./planetmint/backend/tarantool/basic.lua:/opt/tarantool/basic.lua
+    command: tarantool /opt/tarantool/basic.lua
     restart: always
   planetmint:
     depends_on:
-      - mongodb
+      #- mongodb
       - tendermint
+      - tarantool
     build:
       context: .
       dockerfile: Dockerfile-dev
@@ -38,9 +43,9 @@ services:
       - ./pytest.ini:/usr/src/app/pytest.ini
       - ./tox.ini:/usr/src/app/tox.ini
     environment:
-      PLANETMINT_DATABASE_BACKEND: localmongodb
-      PLANETMINT_DATABASE_HOST: mongodb
-      PLANETMINT_DATABASE_PORT: 27017
+      PLANETMINT_DATABASE_BACKEND: tarantooldb 
+      PLANETMINT_DATABASE_HOST: tarantool
+      PLANETMINT_DATABASE_PORT: 3303
       PLANETMINT_SERVER_BIND: 0.0.0.0:9984
       PLANETMINT_WSSERVER_HOST: 0.0.0.0
       PLANETMINT_WSSERVER_ADVERTISED_HOST: planetmint

planetmint/backend/tarantool/connection.py

@@ -16,17 +16,19 @@ class TarantoolDB:
                  reset_database: bool = False):
         self.host = host
         self.port = port
-        self.db_connect = tarantool.connect(host=host, port=port, user=user, password=password)
+        # TODO add user support later on
+        #self.db_connect = tarantool.connect(host=host, port=port, user=user, password=password)
+        self.db_connect = tarantool.connect(host=host, port=port)
         self._load_setup_files()
         if reset_database:
             self.drop_database()
             self.init_database()
 
     def _load_setup_files(self):
-        init_path = Config().get()["database"]["init_config"]["absolute_path"]
-        drop_path = Config().get()["database"]["drop_config"]["absolute_path"]
-        self.drop_commands = self.__read_commands(file_path=init_path)
-        self.init_commands = self.__read_commands(file_path=drop_path)
+        self.init_path = Config().get()["database"]["init_config"]["absolute_path"]
+        self.drop_path = Config().get()["database"]["drop_config"]["absolute_path"]
+        #self.drop_commands = self.__read_commands(file_path=init_path)
+        #self.init_commands = self.__read_commands(file_path=drop_path)
 
     def space(self, space_name: str):
         return self.db_connect.space(space_name)
@@ -41,17 +43,17 @@ class TarantoolDB:
         return commands
 
     def drop_database(self):
-        from planetmint.backend.tarantool.utils import run
+        from planetmint.backend.tarantool.utils import run2
         db_config = Config().get()["database"]
         # drop_config = db_config["drop_config"]
         # f_path = "%s%s" % (drop_config["relative_path"], drop_config["drop_file"])
         # commands = self.__read_commands(file_path=f_path)
-        run(commands=self.drop_commands, config=db_config)
+        run2(commands=self.drop_path, config=db_config)
 
     def init_database(self):
-        from planetmint.backend.tarantool.utils import run
+        from planetmint.backend.tarantool.utils import run2
         db_config = Config().get()["database"]
         # init_config = db_config["init_config"]
         # f_path = "%s%s" % (init_config["relative_path"], init_config["init_file"])
         # commands = self.__read_commands(file_path=f_path)
-        run(commands=self.init_commands, config=db_config)
+        run2(commands=self.init_path, config=db_config)

planetmint/backend/tarantool/utils.py

@@ -2,9 +2,11 @@ import subprocess
 from planetmint.config import Config
 
 
-def run(commands: list, config: dict):
+
+
+def run2(commands: list, config: dict):
     sshProcess = subprocess.Popen(
-        ['%s %s:%s@%s:%s' % (config["service"], config["login"], config["password"], config["host"], config["port"])],
+        ['%s %s:%s < %s' % ("tarantoolctl connect", "localhost", "3303", "planetmint/backend/tarantool/init.lua")],
         stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         universal_newlines=True,
@@ -20,21 +22,41 @@ def run(commands: list, config: dict):
     #  TODO To add here Exception Handler for stdout
     # for line in sshProcess.stdout:
     #     print(line)
-
-
-def __read_commands(file_path):
-    with open(file_path, "r") as cmd_file:
-        commands = [line.strip()+'\n' for line in cmd_file.readlines() if len(str(line)) > 1]
-        cmd_file.close()
-    return commands
-
-
-def _load_setup_files():
-    drop_commands = __read_commands(file_path="planetmint/backend/tarantool/drop.lua")
-    init_commands = __read_commands(file_path="planetmint/backend/tarantool/init.lua")
-    return init_commands, drop_commands
-
-
-init, drop = _load_setup_files()
-db_config = Config().get()["database"]
-run(commands=drop, config=db_config)
+#
+#def run(commands: list, config: dict):
+#    sshProcess = subprocess.Popen(
+#        ['%s %s:%s@%s:%s' % (config["service"], config["login"], config["password"], config["host"], config["port"])],
+#        stdin=subprocess.PIPE,
+#        stdout=subprocess.PIPE,
+#        universal_newlines=True,
+#        bufsize=0,
+#        shell=True)
+#
+#    for cmd in commands:
+#        try:
+#            sshProcess.stdin.write(cmd)
+#        except Exception as cmd_err:
+#            print(str(cmd_err))
+#    sshProcess.stdin.close()
+#    #  TODO To add here Exception Handler for stdout
+#    # for line in sshProcess.stdout:
+#    #     print(line)
+#
+#
+#def __read_commands(file_path):
+#    with open(file_path, "r") as cmd_file:
+#        commands = [line.strip()+'\n' for line in cmd_file.readlines() if len(str(line)) > 1]
+#        cmd_file.close()
+#    return commands
+#
+#
+#def _load_setup_files():
+#    drop_commands = __read_commands(file_path="planetmint/backend/tarantool/drop.lua")
+#    init_commands = __read_commands(file_path="planetmint/backend/tarantool/init.lua")
+#    return init_commands, drop_commands
+#
+#
+#init, drop = _load_setup_files()
+#db_config = Config().get()["database"]
+#run(commands=drop, config=db_config)
+#

planetmint/common/transaction.py

@@ -1123,7 +1123,8 @@ class Transaction(object):
 
     # TODO: This method shouldn't call `_remove_signatures`
     def __str__(self):
-        tx = Transaction._remove_signatures(self.to_dict())
+        _tx = self.to_dict()
+        tx = Transaction._remove_signatures(_tx)
         return Transaction._to_str(tx)
 
     @classmethod
@@ -1182,6 +1183,10 @@ class Transaction(object):
 
         tx_body_serialized = Transaction._to_str(tx_body)
         valid_tx_id = Transaction._to_hash(tx_body_serialized)
+        print( f" valid TX : {valid_tx_id}")
+        print( f" proposed TX id : {proposed_tx_id}")
+        print( f" tx body : {tx_body}")
+        print( f" tx serialized : {tx_body_serialized}")
         if proposed_tx_id != valid_tx_id:
             err_msg = ("The transaction's id '{}' isn't equal to "
                        "the hash of its body, i.e. it's not valid.")
@@ -1200,7 +1205,7 @@ class Transaction(object):
         """
         operation = tx.get('operation', Transaction.CREATE) if isinstance(tx, dict) else Transaction.CREATE
         cls = Transaction.resolve_class(operation)
-
+        print( f" Schema validation {tx}")
         if not skip_schema_validation:
             cls.validate_id(tx)
             cls.validate_schema(tx)

planetmint/config.py

@@ -65,7 +65,7 @@ class Config(metaclass=Singleton):
             'port': 3303,
             "connect_now": True,
             "encoding": "utf-8",
-            "login": "",
+            "login": "guest",
             'password': "",
             "service": "tarantoolctl connect",
             "init_config": self.__private_init_config,

planetmint/core.py

@@ -44,6 +44,7 @@ class App(BaseApplication):
         self.validators = None
         self.new_height = None
         self.chain = self.planetmint.get_latest_abci_chain()
+        print( f"chain: {self.chain}")
         
     def log_abci_migration_error(self, chain_id, validators):
         logger.error('An ABCI chain migration is in process. '
@@ -65,6 +66,7 @@ class App(BaseApplication):
         height = 0
 
         known_chain = self.planetmint.get_latest_abci_chain()
+        print( f" known_chain: {known_chain}")
         if known_chain is not None:
             chain_id = known_chain['chain_id']
 

tests/backend/tarantool/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pytest = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.8"

tests/backend/tarantool/Pipfile.lock

@@ -0,0 +1,78 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "97a0be44f6d5351e166a90d91c789c8100486c7cc30d922ef7f7e3541838acae"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.8"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "attrs": {
+            "hashes": [
+                "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
+                "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
+            ],
+            "version": "==21.4.0"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+            ],
+            "version": "==21.3"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+            ],
+            "version": "==1.0.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+            ],
+            "version": "==1.11.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
+                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+            ],
+            "version": "==3.0.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db",
+                "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171"
+            ],
+            "index": "pypi",
+            "version": "==7.0.1"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+            ],
+            "version": "==2.0.1"
+        }
+    },
+    "develop": {}
+}

tests/backend/tarantool/conftest.py

@@ -1,8 +1,35 @@
 import pytest
-from planetmint.backend import connection
+from planetmint.backend.connection import Connection
+#
+#
+#
+@pytest.fixture
+def dummy_db(request):
+    from planetmint.backend import Connection
 
+    conn = Connection()
+    dbname = request.fixturename
+    xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid')
+    if xdist_suffix:
+        dbname = '{}_{}'.format(dbname, xdist_suffix)
+
+    conn.drop_database()
+    #_drop_db(conn, dbname)  # make sure we start with a clean DB
+    #schema.init_database(conn, dbname)
+    conn.init_database()
+    yield dbname
+
+    conn.drop_database()
+    #_drop_db(conn, dbname)
+
+#def _drop_db(conn, dbname):
+#    try:
+#        conn.drop_database()
+#        schema.drop_database(conn, dbname)
+#    except DatabaseDoesNotExist:
+#        pass
 
 @pytest.fixture
 def db_conn():
-    conn = connection.Connection()
+    conn = Connection()
     return conn

tests/backend/tarantool/test_schema.py

@@ -4,75 +4,68 @@
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
 from planetmint.config import Config
-
-
+from planetmint.backend import Connection
+from planetmint.backend.tarantool.connection import TarantoolDB
 def test_init_database_is_graceful_if_db_exists():
-    import planetmint
-    from planetmint import backend
-    from planetmint.backend.schema import init_database
 
-    conn = backend.connect()
-    dbname = Config().get()['database']['name']
-
-    # The db is set up by the fixtures
-    assert dbname in conn.conn.list_database_names()
-
-    init_database()
     
+    conn = TarantoolDB('localhost', 3303)
+    conn.drop_database()
+    conn.init_database()
 
 def test_create_tables():
-    import planetmint
-    from planetmint import backend
     from planetmint.backend import schema
 
-    conn = backend.connect()
-    dbname = Config().get()['database']['name']
+    conn = TarantoolDB('localhost', 3303)
+    #conn = Connection()
+    #dbname = Config().get()['database']['name']
 
     # The db is set up by the fixtures so we need to remove it
-    conn.conn.drop_database(dbname)
-    schema.create_database(conn, dbname)
-    schema.create_tables(conn, dbname)
+    conn.drop_database()
+    conn.init_database()
 
-    collection_names = conn.conn[dbname].list_collection_names()
-    assert set(collection_names) == {
-        'transactions', 'assets', 'metadata', 'blocks', 'utxos', 'validators', 'elections',
-        'pre_commit', 'abci_chains',
-    }
-
-    indexes = conn.conn[dbname]['assets'].index_information().keys()
-    assert set(indexes) == {'_id_', 'asset_id', 'text'}
-
-    index_info = conn.conn[dbname]['transactions'].index_information()
-    indexes = index_info.keys()
-    assert set(indexes) == {
-        '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'}
-    assert index_info['transaction_id']['unique']
-
-    index_info = conn.conn[dbname]['blocks'].index_information()
-    indexes = index_info.keys()
-    assert set(indexes) == {'_id_', 'height'}
-    assert index_info['height']['unique']
-
-    index_info = conn.conn[dbname]['utxos'].index_information()
-    assert set(index_info.keys()) == {'_id_', 'utxo'}
-    assert index_info['utxo']['unique']
-    assert index_info['utxo']['key'] == [('transaction_id', 1),
-                                         ('output_index', 1)]
-
-    indexes = conn.conn[dbname]['elections'].index_information()
-    assert set(indexes.keys()) == {'_id_', 'election_id_height'}
-    assert indexes['election_id_height']['unique']
-
-    indexes = conn.conn[dbname]['pre_commit'].index_information()
-    assert set(indexes.keys()) == {'_id_', 'height'}
-    assert indexes['height']['unique']
+    # TOTO verify spaces
+    #collection_names = conn.conn[dbname].list_collection_names()
+    #assert set(collection_names) == {
+    #    'transactions', 'assets', 'metadata', 'blocks', 'utxos', 'validators', 'elections',
+    #    'pre_commit', 'abci_chains',
+    #}
+#
+    #indexes = conn.conn[dbname]['assets'].index_information().keys()
+    #assert set(indexes) == {'_id_', 'asset_id', 'text'}
+#
+    #index_info = conn.conn[dbname]['transactions'].index_information()
+    #indexes = index_info.keys()
+    #assert set(indexes) == {
+    #    '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'}
+    #assert index_info['transaction_id']['unique']
+#
+    #index_info = conn.conn[dbname]['blocks'].index_information()
+    #indexes = index_info.keys()
+    #assert set(indexes) == {'_id_', 'height'}
+    #assert index_info['height']['unique']
+#
+    #index_info = conn.conn[dbname]['utxos'].index_information()
+    #assert set(index_info.keys()) == {'_id_', 'utxo'}
+    #assert index_info['utxo']['unique']
+    #assert index_info['utxo']['key'] == [('transaction_id', 1),
+    #                                     ('output_index', 1)]
+#
+    #indexes = conn.conn[dbname]['elections'].index_information()
+    #assert set(indexes.keys()) == {'_id_', 'election_id_height'}
+    #assert indexes['election_id_height']['unique']
+#
+    #indexes = conn.conn[dbname]['pre_commit'].index_information()
+    #assert set(indexes.keys()) == {'_id_', 'height'}
+    #assert indexes['height']['unique']
 
 
 def test_drop(dummy_db):
     from planetmint import backend
     from planetmint.backend import schema
 
-    conn = backend.connect()
-    assert dummy_db in conn.conn.list_database_names()
-    schema.drop_database(conn, dummy_db)
-    assert dummy_db not in conn.conn.list_database_names()
+    conn = TarantoolDB('localhost', 3303)
+    assert assets in conn.spaces('assets')
+    conn.drop_database()
+    assert assets not in conn.conn.list_database_names()
+

tests/commands/test_commands.py

@@ -275,7 +275,8 @@ def test_run_recover(b, alice, bob):
                              asset={'cycle': 'hero'},
                              metadata={'name': 'hohenheim'}) \
                      .sign([bob.private_key])
-
+    print( tx1.id)
+    print( tx2.id)
     # store the transactions
     b.store_bulk_transactions([tx1, tx2])
 

tests/conftest.py

@@ -326,6 +326,7 @@ def dummy_db(request):
     if xdist_suffix:
         dbname = '{}_{}'.format(dbname, xdist_suffix)
 
+    
     _drop_db(conn, dbname)  # make sure we start with a clean DB
     schema.init_database(conn, dbname)
     yield dbname
@@ -335,6 +336,7 @@ def dummy_db(request):
 
 def _drop_db(conn, dbname):
     try:
+        conn.drop_database()
         schema.drop_database(conn, dbname)
     except DatabaseDoesNotExist:
         pass