Planetmint tarantool (#152)

* added indexed_pattern_search to basic lua, implemented text_search

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* adjusted some queries, wip print statements to be removed

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* fixed get_metadata

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added some prints and simplified code

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* fixed __asset_check

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* removed print statements

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added limit to text_Search

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

planetmint/backend/tarantool/basic.lua

@@ -1 +1,78 @@
 box.cfg{listen = 3303}
+
+function indexed_pattern_search(space_name, field_no, pattern)
+  if (box.space[space_name] == nil) then
+    print("Error: Failed to find the specified space")
+    return nil
+  end
+  local index_no = -1
+  for i=0,box.schema.INDEX_MAX,1 do
+    if (box.space[space_name].index[i] == nil) then break end
+    if (box.space[space_name].index[i].type == "TREE"
+        and box.space[space_name].index[i].parts[1].fieldno == field_no
+        and (box.space[space_name].index[i].parts[1].type == "scalar"
+        or box.space[space_name].index[i].parts[1].type == "string")) then
+      index_no = i
+      break
+    end
+  end
+  if (index_no == -1) then
+    print("Error: Failed to find an appropriate index")
+    return nil
+  end
+  local index_search_key = ""
+  local index_search_key_length = 0
+  local last_character = ""
+  local c = ""
+  local c2 = ""
+  for i=1,string.len(pattern),1 do
+    c = string.sub(pattern, i, i)
+    if (last_character ~= "%") then
+      if (c == '^' or c == "$" or c == "(" or c == ")" or c == "."
+                    or c == "[" or c == "]" or c == "*" or c == "+"
+                    or c == "-" or c == "?") then
+        break
+      end
+      if (c == "%") then
+        c2 = string.sub(pattern, i + 1, i + 1)
+        if (string.match(c2, "%p") == nil) then break end
+        index_search_key = index_search_key .. c2
+      else
+        index_search_key = index_search_key .. c
+      end
+    end
+    last_character = c
+  end
+  index_search_key_length = string.len(index_search_key)
+  local result_set = {}
+  local number_of_tuples_in_result_set = 0
+  local previous_tuple_field = ""
+  while true do
+    local number_of_tuples_since_last_yield = 0
+    local is_time_for_a_yield = false
+    for _,tuple in box.space[space_name].index[index_no]:
+    pairs(index_search_key,{iterator = box.index.GE}) do
+      if (string.sub(tuple[field_no], 1, index_search_key_length)
+      > index_search_key) then
+        break
+      end
+      number_of_tuples_since_last_yield = number_of_tuples_since_last_yield + 1
+      if (number_of_tuples_since_last_yield >= 10
+          and tuple[field_no] ~= previous_tuple_field) then
+        index_search_key = tuple[field_no]
+        is_time_for_a_yield = true
+        break
+        end
+      previous_tuple_field = tuple[field_no]
+      if (string.match(tuple[field_no], pattern) ~= nil) then
+        number_of_tuples_in_result_set = number_of_tuples_in_result_set + 1
+        result_set[number_of_tuples_in_result_set] = tuple
+      end
+    end
+    if (is_time_for_a_yield ~= true) then
+      break
+    end
+    require('fiber').yield()
+  end
+  return result_set
+end

planetmint/backend/tarantool/query.py

@@ -9,6 +9,7 @@ from hashlib import sha256
 from operator import itemgetter
 
 import tarantool.error
+import json
 
 from planetmint.backend import query
 from planetmint.backend.utils import module_dispatch_registrar
@@ -109,7 +110,7 @@ def store_metadatas(connection, metadata: list):
     for meta in metadata:
         connection.run(
             connection.space("meta_data").insert(
-                (meta["id"], meta["data"] if not "metadata" in meta else meta["metadata"]))
+                (meta["id"], json.dumps(meta["data"] if not "metadata" in meta else meta["metadata"])))
         )
 
 
@@ -122,33 +123,33 @@ def get_metadata(connection, transaction_ids: list):
         )
         if metadata is not None:
             if len(metadata) > 0:
+                metadata[0] = list(metadata[0])
+                metadata[0][1] = json.loads(metadata[0][1])
+                metadata[0] = tuple(metadata[0])
                 _returned_data.append(metadata)
-    return _returned_data if len(_returned_data) > 0 else None
+    return _returned_data
 
 
 @register_query(TarantoolDBConnection)
 def store_asset(connection, asset):
-    convert = lambda obj: obj if isinstance(obj, tuple) else (obj, obj["id"], obj["id"])
+    def convert(obj):
-    try:
+        if isinstance(obj, tuple):
+            obj = list(obj)
+            obj[0] = json.dumps(obj[0])
+            return tuple(obj)
+        else:
+            (json.dumps(obj), obj["id"], obj["id"])
+
     return connection.run(
         connection.space("assets").insert(convert(asset)),
         only_data=False
     )
-    except tarantool.error.DatabaseError:
-        pass
 
 
 @register_query(TarantoolDBConnection)
 def store_assets(connection, assets: list):
-    convert = lambda obj: obj if isinstance(obj, tuple) else (obj, obj["id"], obj["id"])
     for asset in assets:
-        try:
+        store_asset(connection, asset)
-            connection.run(
-                connection.space("assets").insert(convert(asset)),
-                only_data=False
-            )
-        except tarantool.error.DatabaseError:
-            pass
 
 
 @register_query(TarantoolDBConnection)
@@ -156,7 +157,8 @@ def get_asset(connection, asset_id: str):
     _data = connection.run(
         connection.space("assets").select(asset_id, index="txid_search")
     )
-    return _data[0][0] if len(_data) > 0 else []
+
+    return json.loads(_data[0][0]) if len(_data) > 0 else []
 
 
 @register_query(TarantoolDBConnection)
@@ -165,6 +167,7 @@ def get_assets(connection, assets_ids: list) -> list:
     for _id in list(set(assets_ids)):
         asset = get_asset(connection, _id)
         _returned_data.append(asset)
+
     return sorted(_returned_data, key=lambda k: k["id"], reverse=False)
 
 
@@ -254,27 +257,13 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None,
 
     return tuple([elem[0] for elem in _transactions])
 
-
+@register_query(TarantoolDBConnection)
-# @register_query(TarantoolDB)
+def text_search(conn, search, table='assets', limit=0):
-# def text_search(conn, search, *, language='english', case_sensitive=False,
+    pattern = ".{}.".format(search)
-#                 # TODO review text search in tarantool (maybe, remove)
+    res = conn.run(
-#                 diacritic_sensitive=False, text_score=False, limit=0, table='assets'):
+        conn.space(table).call('indexed_pattern_search', (table, 1, pattern))
-#     cursor = conn.run(
+    )
-#         conn.collection(table)
+    return res[0] if limit == 0 else res[0][:limit]
-#             .find({'$text': {
-#             '$search': search,
-#             '$language': language,
-#             '$caseSensitive': case_sensitive,
-#             '$diacriticSensitive': diacritic_sensitive}},
-#             {'score': {'$meta': 'textScore'}, '_id': False})
-#             .sort([('score', {'$meta': 'textScore'})])
-#             .limit(limit))
-#
-#     if text_score:
-#         return cursor
-#
-#     return (_remove_text_score(obj) for obj in cursor)
-
 
 def _remove_text_score(asset):
     asset.pop('score', None)

planetmint/backend/tarantool/schema.py

@@ -39,7 +39,8 @@ INDEX_COMMANDS = {
         {
             "txid_search": "assets:create_index('txid_search', {type='hash', parts={'tx_id'}})",
             "assetid_search": "assets:create_index('assetid_search', {type='tree',unique=false, parts={'asset_id', 'tx_id'}})",
-            "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})"
+            "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})",
+            "text_search": "assets:create_index('secondary', {unique=false,parts={1,'string'}})"
         },
     "blocks":
         {
@@ -60,7 +61,8 @@ INDEX_COMMANDS = {
         },
     "meta_data":
         {
-            "id_search": "meta_datas:create_index('id_search', { type='hash' , parts={'transaction_id'}})"
+            "id_search": "meta_datas:create_index('id_search', { type='hash' , parts={'transaction_id'}})",
+            "text_search": "meta_datas:create_index('secondary', {unique=false,parts={2,'string'}}"
         },
     "pre_commits":
         {
@@ -107,13 +109,13 @@ SCHEMA_COMMANDS = {
     "abci_chains":
         "abci_chains:format({{name='height' , type='integer'},{name='is_synched' , type='boolean'},{name='chain_id',type='string'}, {name='id', type='string'}})",
     "assets":
-        "assets:format({{name='data' , type='any'}, {name='tx_id', type='string'}, {name='asset_id', type='string'}})",
+        "assets:format({{name='data' , type='string'}, {name='tx_id', type='string'}, {name='asset_id', type='string'}})",
     "blocks":
         "blocks:format{{name='app_hash',type='string'},{name='height' , type='integer'},{name='block_id' , type='string'}}",
     "blocks_tx": "blocks_tx:format{{name='transaction_id', type = 'string'}, {name = 'block_id', type = 'string'}}",
     "elections":
         "elections:format({{name='election_id' , type='string'},{name='height' , type='integer'}, {name='is_concluded' , type='boolean'}})",
-    "meta_data": "meta_datas:format({{name='transaction_id' , type='string'}, {name='meta_data' , type='any'}})",
+    "meta_data": "meta_datas:format({{name='transaction_id' , type='string'}, {name='meta_data' , type='string'}})",
     "pre_commits":
         "pre_commits:format({{name='commit_id', type='string'}, {name='height',type='integer'}, {name='transactions',type=any}})",
     "validators":

planetmint/backend/tarantool/transaction/tools.py

@@ -1,5 +1,6 @@
 from secrets import token_hex
 import copy
+import json
 from planetmint.transactions.common.memoize import HDict
 
 
@@ -56,14 +57,14 @@ class TransactionDecompose:
         if metadata is None:
             return
 
-        self._tuple_transaction["metadata"] = (self._transaction["id"], metadata)
+        self._tuple_transaction["metadata"] = (self._transaction["id"], json.dumps(metadata))
 
     def __asset_check(self):
         _asset = self._transaction.get("asset")
         if _asset is None:
             return
         asset_id = _asset["id"] if _asset.get("id") is not None else self._transaction["id"]
-        self._tuple_transaction["asset"] = (_asset, self._transaction["id"], asset_id)
+        self._tuple_transaction["asset"] = (json.dumps(_asset), self._transaction["id"], asset_id)
 
     def __prepare_inputs(self):
         _inputs = []
@@ -154,10 +155,10 @@ class TransactionCompose:
     def _get_asset(self):
         _asset = iter(self.db_results["asset"])
         _res_asset = next(iter(next(_asset, iter([]))), None)
-        return _res_asset
+        return json.loads(_res_asset)
 
     def _get_metadata(self):
-        return self.db_results["metadata"][0][1] if len(self.db_results["metadata"]) == 1 else None
+        return json.loads(self.db_results["metadata"][0][1]) if len(self.db_results["metadata"]) == 1 else None
 
     def _get_inputs(self):
         _inputs = []

planetmint/web/views/metadata.py

@@ -42,7 +42,7 @@ class MetadataApi(Resource):
         pool = current_app.config['bigchain_pool']
 
         with pool() as planet:
-            args['table'] = 'metadata'
+            args['table'] = 'meta_data'
             metadata = planet.text_search(**args)
 
         try: