From 93f282dd8cdc73f89fab09380a422feffa1df112 Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 14 Jun 2022 08:00:34 +0200 Subject: [PATCH 01/11] added indexed_pattern_search to basic lua, implemented text_search Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/basic.lua | 77 ++++++++++++++++++++++++++ planetmint/backend/tarantool/query.py | 7 +++ planetmint/backend/tarantool/schema.py | 5 +- 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/planetmint/backend/tarantool/basic.lua b/planetmint/backend/tarantool/basic.lua index bdea22b..fcc46eb 100644 --- a/planetmint/backend/tarantool/basic.lua +++ b/planetmint/backend/tarantool/basic.lua @@ -1 +1,78 @@ box.cfg{listen = 3303} + +function indexed_pattern_search(space_name, field_no, pattern) + if (box.space[space_name] == nil) then + print("Error: Failed to find the specified space") + return nil + end + local index_no = -1 + for i=0,box.schema.INDEX_MAX,1 do + if (box.space[space_name].index[i] == nil) then break end + if (box.space[space_name].index[i].type == "TREE" + and box.space[space_name].index[i].parts[1].fieldno == field_no + and (box.space[space_name].index[i].parts[1].type == "scalar" + or box.space[space_name].index[i].parts[1].type == "string")) then + index_no = i + break + end + end + if (index_no == -1) then + print("Error: Failed to find an appropriate index") + return nil + end + local index_search_key = "" + local index_search_key_length = 0 + local last_character = "" + local c = "" + local c2 = "" + for i=1,string.len(pattern),1 do + c = string.sub(pattern, i, i) + if (last_character ~= "%") then + if (c == '^' or c == "$" or c == "(" or c == ")" or c == "." + or c == "[" or c == "]" or c == "*" or c == "+" + or c == "-" or c == "?") then + break + end + if (c == "%") then + c2 = string.sub(pattern, i + 1, i + 1) + if (string.match(c2, "%p") == nil) then break end + index_search_key = index_search_key .. c2 + else + index_search_key = index_search_key .. c + end + end + last_character = c + end + index_search_key_length = string.len(index_search_key) + local result_set = {} + local number_of_tuples_in_result_set = 0 + local previous_tuple_field = "" + while true do + local number_of_tuples_since_last_yield = 0 + local is_time_for_a_yield = false + for _,tuple in box.space[space_name].index[index_no]: + pairs(index_search_key,{iterator = box.index.GE}) do + if (string.sub(tuple[field_no], 1, index_search_key_length) + > index_search_key) then + break + end + number_of_tuples_since_last_yield = number_of_tuples_since_last_yield + 1 + if (number_of_tuples_since_last_yield >= 10 + and tuple[field_no] ~= previous_tuple_field) then + index_search_key = tuple[field_no] + is_time_for_a_yield = true + break + end + previous_tuple_field = tuple[field_no] + if (string.match(tuple[field_no], pattern) ~= nil) then + number_of_tuples_in_result_set = number_of_tuples_in_result_set + 1 + result_set[number_of_tuples_in_result_set] = tuple + end + end + if (is_time_for_a_yield ~= true) then + break + end + require('fiber').yield() + end + return result_set +end \ No newline at end of file diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 6a1b5b0..f9446ab 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -275,6 +275,13 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, # # return (_remove_text_score(obj) for obj in cursor) +@register_query(TarantoolDBConnection) +def text_search(conn, search, table='assets'): + pattern = ".{}.".format(search) + res = conn.run( + conn.space(table).call('indexed_pattern_search', (table, 1, pattern)) + ) + return res def _remove_text_score(asset): asset.pop('score', None) diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index 73b13dc..de3b384 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -39,7 +39,8 @@ INDEX_COMMANDS = { { "txid_search": "assets:create_index('txid_search', {type='hash', parts={'tx_id'}})", "assetid_search": "assets:create_index('assetid_search', {type='tree',unique=false, parts={'asset_id', 'tx_id'}})", - "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})" + "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})", + "secondary": "assets:create_index('secondary', {unique=false,parts={1,'string',2,'string',3,'string'}})" }, "blocks": { @@ -107,7 +108,7 @@ SCHEMA_COMMANDS = { "abci_chains": "abci_chains:format({{name='height' , type='integer'},{name='is_synched' , type='boolean'},{name='chain_id',type='string'}, {name='id', type='string'}})", "assets": - "assets:format({{name='data' , type='any'}, {name='tx_id', type='string'}, {name='asset_id', type='string'}})", + "assets:format({{name='data' , type='string'}, {name='tx_id', type='string'}, {name='asset_id', type='string'}})", "blocks": "blocks:format{{name='app_hash',type='string'},{name='height' , type='integer'},{name='block_id' , type='string'}}", "blocks_tx": "blocks_tx:format{{name='transaction_id', type = 'string'}, {name = 'block_id', type = 'string'}}", From e1a7ade68cf5a6ed2fe44e23f72c904ef372ef4f Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 14 Jun 2022 17:15:05 +0200 Subject: [PATCH 02/11] adjusted some queries, wip print statements to be removed Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 50 +++++++++++-------- planetmint/backend/tarantool/schema.py | 7 +-- .../backend/tarantool/transaction/tools.py | 12 +++-- planetmint/web/views/metadata.py | 2 +- 4 files changed, 43 insertions(+), 28 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index f9446ab..5fbc1a3 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -9,6 +9,7 @@ from hashlib import sha256 from operator import itemgetter import tarantool.error +import json from planetmint.backend import query from planetmint.backend.utils import module_dispatch_registrar @@ -34,6 +35,9 @@ def _group_transaction_by_ids(connection, txids: list): _txassets = connection.run(connection.space("assets").select(txid, index="txid_search")) _txmeta = connection.run(connection.space("meta_data").select(txid, index="id_search")) + print('TX ASSETS') + print(_txassets) + _txinputs = sorted(_txinputs, key=itemgetter(6), reverse=False) _txoutputs = sorted(_txoutputs, key=itemgetter(8), reverse=False) result_map = { @@ -109,7 +113,7 @@ def store_metadatas(connection, metadata: list): for meta in metadata: connection.run( connection.space("meta_data").insert( - (meta["id"], meta["data"] if not "metadata" in meta else meta["metadata"])) + (meta["id"], json.dumps(meta["data"] if not "metadata" in meta else meta["metadata"]))) ) @@ -122,33 +126,30 @@ def get_metadata(connection, transaction_ids: list): ) if metadata is not None: if len(metadata) > 0: - _returned_data.append(metadata) + _returned_data.append(json.loads(metadata)) return _returned_data if len(_returned_data) > 0 else None @register_query(TarantoolDBConnection) def store_asset(connection, asset): - convert = lambda obj: obj if isinstance(obj, tuple) else (obj, obj["id"], obj["id"]) - try: - return connection.run( - connection.space("assets").insert(convert(asset)), - only_data=False - ) - except tarantool.error.DatabaseError: - pass + def convert(obj): + if isinstance(obj, tuple): + obj = list(obj) + obj[0] = json.dumps(obj[0]) + return tuple(obj) + else: + (json.dumps(obj), obj["id"], obj["id"]) + + return connection.run( + connection.space("assets").insert(convert(asset)), + only_data=False + ) @register_query(TarantoolDBConnection) def store_assets(connection, assets: list): - convert = lambda obj: obj if isinstance(obj, tuple) else (obj, obj["id"], obj["id"]) for asset in assets: - try: - connection.run( - connection.space("assets").insert(convert(asset)), - only_data=False - ) - except tarantool.error.DatabaseError: - pass + store_asset(connection, asset) @register_query(TarantoolDBConnection) @@ -156,7 +157,12 @@ def get_asset(connection, asset_id: str): _data = connection.run( connection.space("assets").select(asset_id, index="txid_search") ) - return _data[0][0] if len(_data) > 0 else [] + + print('GET ASSET') + + print(_data) + + return json.loads(_data[0][0]) if len(_data) > 0 else [] @register_query(TarantoolDBConnection) @@ -165,6 +171,10 @@ def get_assets(connection, assets_ids: list) -> list: for _id in list(set(assets_ids)): asset = get_asset(connection, _id) _returned_data.append(asset) + + print('############# RETURNED DATA ##########') + print(_returned_data) + return sorted(_returned_data, key=lambda k: k["id"], reverse=False) @@ -276,7 +286,7 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, # return (_remove_text_score(obj) for obj in cursor) @register_query(TarantoolDBConnection) -def text_search(conn, search, table='assets'): +def text_search(conn, search, table='assets', limit=0): pattern = ".{}.".format(search) res = conn.run( conn.space(table).call('indexed_pattern_search', (table, 1, pattern)) diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index de3b384..6ef03b4 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -37,10 +37,10 @@ INDEX_COMMANDS = { }, "assets": { + "secondary": "assets:create_index('secondary', {unique=false,parts={1,'string',2,'string',3,'string'}})", "txid_search": "assets:create_index('txid_search', {type='hash', parts={'tx_id'}})", "assetid_search": "assets:create_index('assetid_search', {type='tree',unique=false, parts={'asset_id', 'tx_id'}})", - "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})", - "secondary": "assets:create_index('secondary', {unique=false,parts={1,'string',2,'string',3,'string'}})" + "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})" }, "blocks": { @@ -61,6 +61,7 @@ INDEX_COMMANDS = { }, "meta_data": { + "secondary": "meta_datas:create_index('secondary', {unique=false,parts={1,'string',2,'string'}}", "id_search": "meta_datas:create_index('id_search', { type='hash' , parts={'transaction_id'}})" }, "pre_commits": @@ -114,7 +115,7 @@ SCHEMA_COMMANDS = { "blocks_tx": "blocks_tx:format{{name='transaction_id', type = 'string'}, {name = 'block_id', type = 'string'}}", "elections": "elections:format({{name='election_id' , type='string'},{name='height' , type='integer'}, {name='is_concluded' , type='boolean'}})", - "meta_data": "meta_datas:format({{name='transaction_id' , type='string'}, {name='meta_data' , type='any'}})", + "meta_data": "meta_datas:format({{name='transaction_id' , type='string'}, {name='meta_data' , type='string'}})", "pre_commits": "pre_commits:format({{name='commit_id', type='string'}, {name='height',type='integer'}, {name='transactions',type=any}})", "validators": diff --git a/planetmint/backend/tarantool/transaction/tools.py b/planetmint/backend/tarantool/transaction/tools.py index 28fea32..5e7371a 100644 --- a/planetmint/backend/tarantool/transaction/tools.py +++ b/planetmint/backend/tarantool/transaction/tools.py @@ -1,5 +1,6 @@ from secrets import token_hex import copy +import json from planetmint.transactions.common.memoize import HDict @@ -56,14 +57,14 @@ class TransactionDecompose: if metadata is None: return - self._tuple_transaction["metadata"] = (self._transaction["id"], metadata) + self._tuple_transaction["metadata"] = (self._transaction["id"], json.dumps(metadata)) def __asset_check(self): _asset = self._transaction.get("asset") if _asset is None: return asset_id = _asset["id"] if _asset.get("id") is not None else self._transaction["id"] - self._tuple_transaction["asset"] = (_asset, self._transaction["id"], asset_id) + self._tuple_transaction["asset"] = (json.loads(_asset), self._transaction["id"], asset_id) def __prepare_inputs(self): _inputs = [] @@ -152,12 +153,15 @@ class TransactionCompose: return self.db_results["transaction"][0] def _get_asset(self): + print('_GET_ASSET') _asset = iter(self.db_results["asset"]) + print(_asset) _res_asset = next(iter(next(_asset, iter([]))), None) - return _res_asset + print(_res_asset) + return json.loads(_res_asset) def _get_metadata(self): - return self.db_results["metadata"][0][1] if len(self.db_results["metadata"]) == 1 else None + return json.loads(self.db_results["metadata"][0][1]) if len(self.db_results["metadata"]) == 1 else None def _get_inputs(self): _inputs = [] diff --git a/planetmint/web/views/metadata.py b/planetmint/web/views/metadata.py index cf1a61d..d8faf19 100644 --- a/planetmint/web/views/metadata.py +++ b/planetmint/web/views/metadata.py @@ -42,7 +42,7 @@ class MetadataApi(Resource): pool = current_app.config['bigchain_pool'] with pool() as planet: - args['table'] = 'metadata' + args['table'] = 'meta_data' metadata = planet.text_search(**args) try: From 54d4f31929970477625b5ac63acc8956d145e9c9 Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 14 Jun 2022 23:28:33 +0200 Subject: [PATCH 03/11] fixed get_metadata Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 5fbc1a3..481b65d 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -126,7 +126,10 @@ def get_metadata(connection, transaction_ids: list): ) if metadata is not None: if len(metadata) > 0: - _returned_data.append(json.loads(metadata)) + metadata[0] = list(metadata[0]) + metadata[0][1] = json.loads(metadata[0][1]) + metadata[0] = tuple(metadata[0]) + _returned_data.append(metadata) return _returned_data if len(_returned_data) > 0 else None From 1153f1bd6f298fafbbbc25dd09f8957c7d99eabb Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Wed, 15 Jun 2022 10:55:47 +0200 Subject: [PATCH 04/11] added some prints and simplified code Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 25 +++---------------------- planetmint/backend/tarantool/schema.py | 8 ++++---- planetmint/web/views/metadata.py | 3 +++ 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 481b65d..5d55d48 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -130,7 +130,7 @@ def get_metadata(connection, transaction_ids: list): metadata[0][1] = json.loads(metadata[0][1]) metadata[0] = tuple(metadata[0]) _returned_data.append(metadata) - return _returned_data if len(_returned_data) > 0 else None + return _returned_data @register_query(TarantoolDBConnection) @@ -267,33 +267,14 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, return tuple([elem[0] for elem in _transactions]) - -# @register_query(TarantoolDB) -# def text_search(conn, search, *, language='english', case_sensitive=False, -# # TODO review text search in tarantool (maybe, remove) -# diacritic_sensitive=False, text_score=False, limit=0, table='assets'): -# cursor = conn.run( -# conn.collection(table) -# .find({'$text': { -# '$search': search, -# '$language': language, -# '$caseSensitive': case_sensitive, -# '$diacriticSensitive': diacritic_sensitive}}, -# {'score': {'$meta': 'textScore'}, '_id': False}) -# .sort([('score', {'$meta': 'textScore'})]) -# .limit(limit)) -# -# if text_score: -# return cursor -# -# return (_remove_text_score(obj) for obj in cursor) - @register_query(TarantoolDBConnection) def text_search(conn, search, table='assets', limit=0): pattern = ".{}.".format(search) + print("TEXT SEARCH FOR: {}".format(search)) res = conn.run( conn.space(table).call('indexed_pattern_search', (table, 1, pattern)) ) + print("TEXT SEARCH RES: {}".format(res)) return res def _remove_text_score(asset): diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index 6ef03b4..bf7017b 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -37,10 +37,10 @@ INDEX_COMMANDS = { }, "assets": { - "secondary": "assets:create_index('secondary', {unique=false,parts={1,'string',2,'string',3,'string'}})", "txid_search": "assets:create_index('txid_search', {type='hash', parts={'tx_id'}})", "assetid_search": "assets:create_index('assetid_search', {type='tree',unique=false, parts={'asset_id', 'tx_id'}})", - "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})" + "only_asset_search": "assets:create_index('only_asset_search', {type='tree', unique=false, parts={'asset_id'}})", + "text_search": "assets:create_index('secondary', {unique=false,parts={1,'string'}})" }, "blocks": { @@ -61,8 +61,8 @@ INDEX_COMMANDS = { }, "meta_data": { - "secondary": "meta_datas:create_index('secondary', {unique=false,parts={1,'string',2,'string'}}", - "id_search": "meta_datas:create_index('id_search', { type='hash' , parts={'transaction_id'}})" + "id_search": "meta_datas:create_index('id_search', { type='hash' , parts={'transaction_id'}})", + "text_search": "meta_datas:create_index('secondary', {unique=false,parts={2,'string'}}" }, "pre_commits": { diff --git a/planetmint/web/views/metadata.py b/planetmint/web/views/metadata.py index d8faf19..35979f8 100644 --- a/planetmint/web/views/metadata.py +++ b/planetmint/web/views/metadata.py @@ -45,6 +45,9 @@ class MetadataApi(Resource): args['table'] = 'meta_data' metadata = planet.text_search(**args) + print('########### METADATA API ############') + print(metadata) + try: # This only works with MongoDB as the backend return list(metadata) From d87c53aa8b3c5aa2b3049a39d5694e11e5ca6c53 Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Mon, 20 Jun 2022 14:24:04 +0200 Subject: [PATCH 05/11] fixed __asset_check Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/transaction/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetmint/backend/tarantool/transaction/tools.py b/planetmint/backend/tarantool/transaction/tools.py index 5e7371a..6815fbe 100644 --- a/planetmint/backend/tarantool/transaction/tools.py +++ b/planetmint/backend/tarantool/transaction/tools.py @@ -64,7 +64,7 @@ class TransactionDecompose: if _asset is None: return asset_id = _asset["id"] if _asset.get("id") is not None else self._transaction["id"] - self._tuple_transaction["asset"] = (json.loads(_asset), self._transaction["id"], asset_id) + self._tuple_transaction["asset"] = (json.dumps(_asset), self._transaction["id"], asset_id) def __prepare_inputs(self): _inputs = [] From e5c62824902e218d10512ef80dfe0595e354c23f Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Mon, 20 Jun 2022 14:29:24 +0200 Subject: [PATCH 06/11] removed print statements Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 10 ---------- planetmint/backend/tarantool/transaction/tools.py | 3 --- planetmint/web/views/metadata.py | 3 --- 3 files changed, 16 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 5d55d48..bfa59ff 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -35,9 +35,6 @@ def _group_transaction_by_ids(connection, txids: list): _txassets = connection.run(connection.space("assets").select(txid, index="txid_search")) _txmeta = connection.run(connection.space("meta_data").select(txid, index="id_search")) - print('TX ASSETS') - print(_txassets) - _txinputs = sorted(_txinputs, key=itemgetter(6), reverse=False) _txoutputs = sorted(_txoutputs, key=itemgetter(8), reverse=False) result_map = { @@ -160,10 +157,6 @@ def get_asset(connection, asset_id: str): _data = connection.run( connection.space("assets").select(asset_id, index="txid_search") ) - - print('GET ASSET') - - print(_data) return json.loads(_data[0][0]) if len(_data) > 0 else [] @@ -175,9 +168,6 @@ def get_assets(connection, assets_ids: list) -> list: asset = get_asset(connection, _id) _returned_data.append(asset) - print('############# RETURNED DATA ##########') - print(_returned_data) - return sorted(_returned_data, key=lambda k: k["id"], reverse=False) diff --git a/planetmint/backend/tarantool/transaction/tools.py b/planetmint/backend/tarantool/transaction/tools.py index 6815fbe..998a742 100644 --- a/planetmint/backend/tarantool/transaction/tools.py +++ b/planetmint/backend/tarantool/transaction/tools.py @@ -153,11 +153,8 @@ class TransactionCompose: return self.db_results["transaction"][0] def _get_asset(self): - print('_GET_ASSET') _asset = iter(self.db_results["asset"]) - print(_asset) _res_asset = next(iter(next(_asset, iter([]))), None) - print(_res_asset) return json.loads(_res_asset) def _get_metadata(self): diff --git a/planetmint/web/views/metadata.py b/planetmint/web/views/metadata.py index 35979f8..d8faf19 100644 --- a/planetmint/web/views/metadata.py +++ b/planetmint/web/views/metadata.py @@ -45,9 +45,6 @@ class MetadataApi(Resource): args['table'] = 'meta_data' metadata = planet.text_search(**args) - print('########### METADATA API ############') - print(metadata) - try: # This only works with MongoDB as the backend return list(metadata) From 772b28eabce80f3034a1afd6c5ac3d3eb29b7c80 Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Mon, 20 Jun 2022 15:51:14 +0200 Subject: [PATCH 07/11] added limit to text_Search Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index bfa59ff..784a17c 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -260,12 +260,10 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, @register_query(TarantoolDBConnection) def text_search(conn, search, table='assets', limit=0): pattern = ".{}.".format(search) - print("TEXT SEARCH FOR: {}".format(search)) res = conn.run( conn.space(table).call('indexed_pattern_search', (table, 1, pattern)) ) - print("TEXT SEARCH RES: {}".format(res)) - return res + return res[0] if limit == 0 else res[0][:limit] def _remove_text_score(asset): asset.pop('score', None) From 1fb42242d7c0555ec93d1dce7579d3cecb21d789 Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 21 Jun 2022 11:22:46 +0200 Subject: [PATCH 08/11] adjusted store_asset behaviour to match mongodb implementation Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 784a17c..32bd560 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -7,10 +7,10 @@ from secrets import token_hex from hashlib import sha256 from operator import itemgetter - -import tarantool.error import json +from tarantool.error import DatabaseError + from planetmint.backend import query from planetmint.backend.utils import module_dispatch_registrar from planetmint.backend.tarantool.connection import TarantoolDBConnection @@ -138,12 +138,14 @@ def store_asset(connection, asset): obj[0] = json.dumps(obj[0]) return tuple(obj) else: - (json.dumps(obj), obj["id"], obj["id"]) - - return connection.run( - connection.space("assets").insert(convert(asset)), - only_data=False - ) + return (json.dumps(obj), obj["id"], obj["id"]) + try: + return connection.run( + connection.space("assets").insert(convert(asset)), + only_data=False + ) + except DatabaseError: + pass @register_query(TarantoolDBConnection) From 697befbbda1ab56c81a52b76501e87a672c1975f Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 21 Jun 2022 11:33:06 +0200 Subject: [PATCH 09/11] fixed test_write_metadata test case Signed-off-by: Lorenz Herzberger --- tests/backend/tarantool/test_queries.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/backend/tarantool/test_queries.py b/tests/backend/tarantool/test_queries.py index 27a0684..8788d47 100644 --- a/tests/backend/tarantool/test_queries.py +++ b/tests/backend/tarantool/test_queries.py @@ -6,16 +6,10 @@ from copy import deepcopy import pytest -import tarantool +import json from planetmint.transactions.types.assets.create import Create from planetmint.transactions.types.assets.transfer import Transfer -# import pymongo - -# # from planetmint.backend.connection import Connection, query -# from planetmint.backend.connection import Connection -# conn = Connection(backend="tarantool_db").get_connection() - pytestmark = pytest.mark.bdb @@ -167,9 +161,7 @@ def test_text_search(table): def test_write_metadata(db_conn): - # from planetmint.backend.connection import Connection from planetmint.backend.tarantool import query - # conn = Connection().get_connection() metadata = [ {'id': "1", 'data': '1'}, @@ -183,7 +175,7 @@ def test_write_metadata(db_conn): metadatas = [] for meta in metadata: _data = db_conn.run(db_conn.space("meta_data").select(meta["id"]))[0] - metadatas.append({"id": _data[0], "data": _data[1]}) + metadatas.append({"id": _data[0], "data": json.loads(_data[1])}) metadatas = sorted(metadatas, key=lambda k: k["id"]) From 33769128710220b526b944143c621cd69ff798b6 Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 21 Jun 2022 15:24:59 +0200 Subject: [PATCH 10/11] fixed pattern search for meta_data Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 3 ++- planetmint/backend/tarantool/schema.py | 2 +- planetmint/web/views/metadata.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 32bd560..499c1c3 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -262,8 +262,9 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, @register_query(TarantoolDBConnection) def text_search(conn, search, table='assets', limit=0): pattern = ".{}.".format(search) + field_no = 1 if table == 'assets' else 2 # 2 for meta_data res = conn.run( - conn.space(table).call('indexed_pattern_search', (table, 1, pattern)) + conn.space(table).call('indexed_pattern_search', (table, field_no, pattern)) ) return res[0] if limit == 0 else res[0][:limit] diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index bf7017b..8f74084 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -62,7 +62,7 @@ INDEX_COMMANDS = { "meta_data": { "id_search": "meta_datas:create_index('id_search', { type='hash' , parts={'transaction_id'}})", - "text_search": "meta_datas:create_index('secondary', {unique=false,parts={2,'string'}}" + "text_search": "meta_datas:create_index('secondary', {unique=false,parts={2,'string'}})" }, "pre_commits": { diff --git a/planetmint/web/views/metadata.py b/planetmint/web/views/metadata.py index d8faf19..171c258 100644 --- a/planetmint/web/views/metadata.py +++ b/planetmint/web/views/metadata.py @@ -46,7 +46,6 @@ class MetadataApi(Resource): metadata = planet.text_search(**args) try: - # This only works with MongoDB as the backend return list(metadata) except OperationError as e: return make_error( From fb0b3121bf0db3e70cf61bc89b8521fd2c04c6eb Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Tue, 21 Jun 2022 17:12:31 +0200 Subject: [PATCH 11/11] fixed text_search result conversion Signed-off-by: Lorenz Herzberger --- planetmint/backend/tarantool/query.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 499c1c3..c25519a 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -266,7 +266,24 @@ def text_search(conn, search, table='assets', limit=0): res = conn.run( conn.space(table).call('indexed_pattern_search', (table, field_no, pattern)) ) - return res[0] if limit == 0 else res[0][:limit] + + to_return = [] + + if len(res[0]): # NEEDS BEAUTIFICATION + if table == 'assets': + for result in res[0]: + to_return.append({ + 'data': json.loads(result[0])['data'], + 'id': result[1] + }) + else: + for result in res[0]: + to_return.append({ + 'metadata': json.loads(result[1]), + 'id': result[0] + }) + + return to_return if limit == 0 else to_return[:limit] def _remove_text_score(asset): asset.pop('score', None)