From 2302673d8ee3b328c548db594042ace81dfc05de Mon Sep 17 00:00:00 2001 From: Lorenz Herzberger Date: Mon, 17 Apr 2023 20:08:52 +0200 Subject: [PATCH] adjusted tarantool scripts for use in service Signed-off-by: Lorenz Herzberger --- CHANGELOG.md | 3 + docker-compose.yml | 2 - planetmint/backend/schema.py | 16 +--- .../backend/tarantool/opt/functions.lua | 20 ----- planetmint/backend/tarantool/opt/init.lua | 68 ++++++++++++++-- .../backend/tarantool/opt/migrations.lua | 79 ------------------- .../backend/tarantool/sync_io/schema.py | 9 +-- planetmint/commands/planetmint.py | 10 +-- pyproject.toml | 2 +- 9 files changed, 69 insertions(+), 140 deletions(-) delete mode 100644 planetmint/backend/tarantool/opt/functions.lua delete mode 100644 planetmint/backend/tarantool/opt/migrations.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 213f209..3e4faea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ For reference, the possible headings are: * **Known Issues** * **Notes** +## [2.4.2] - 2023-17-04 +* **Fixed** fixed migration behaviour for non docker service + ## [2.4.2] - 2023-13-04 * **Added** planetmint migration commands diff --git a/docker-compose.yml b/docker-compose.yml index 6bfa5ff..a030d4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,8 +23,6 @@ services: - "8081:8081" volumes: - ./planetmint/backend/tarantool/opt/init.lua:/opt/tarantool/init.lua - - ./planetmint/backend/tarantool/opt/functions.lua:/opt/tarantool/functions.lua - - ./planetmint/backend/tarantool/opt/migrations.lua:/opt/tarantool/migrations.lua entrypoint: tarantool /opt/tarantool/init.lua restart: always planetmint: diff --git a/planetmint/backend/schema.py b/planetmint/backend/schema.py index 2af754f..676217b 100644 --- a/planetmint/backend/schema.py +++ b/planetmint/backend/schema.py @@ -138,20 +138,8 @@ def init_database(connection, dbname): @singledispatch -def migrate_up(connection): - """Migrate database up - - Args: - connection (:class:`~planetmint.backend.connection.Connection`): an - existing connection to use to migrate the database. - Creates one if not given. - """ - raise NotImplementedError - - -@singledispatch -def migrate_down(connection): - """Migrate database down +def migrate(connection): + """Migrate database Args: connection (:class:`~planetmint.backend.connection.Connection`): an diff --git a/planetmint/backend/tarantool/opt/functions.lua b/planetmint/backend/tarantool/opt/functions.lua deleted file mode 100644 index 24a2d5c..0000000 --- a/planetmint/backend/tarantool/opt/functions.lua +++ /dev/null @@ -1,20 +0,0 @@ -local fiber = require('fiber') - -local export = {} - -function export.atomic(batch_size, iter, fn) - box.atomic(function() - local i = 0 - for _, x in iter:unwrap() do - fn(x) - i = i + 1 - if i % batch_size == 0 then - box.commit() - fiber.yield() -- for read-only operations when `commit` doesn't yield - box.begin() - end - end - end) -end - -return export \ No newline at end of file diff --git a/planetmint/backend/tarantool/opt/init.lua b/planetmint/backend/tarantool/opt/init.lua index 7dac85d..7a61e73 100644 --- a/planetmint/backend/tarantool/opt/init.lua +++ b/planetmint/backend/tarantool/opt/init.lua @@ -1,4 +1,4 @@ -local migrations = require('migrations') +local fiber = require('fiber') box.cfg{listen = 3303} @@ -334,12 +334,64 @@ function delete_output( id ) box.space.outputs:delete(id) end -function migrate_up() - migrations.update_utxo_13042023.up() - -- add newer migrations below +function atomic(batch_size, iter, fn) + box.atomic(function() + local i = 0 + for _, x in iter:unwrap() do + fn(x) + i = i + 1 + if i % batch_size == 0 then + box.commit() + fiber.yield() -- for read-only operations when `commit` doesn't yield + box.begin() + end + end + end) end -function migrate_down() - -- add newer migrations above - migrations.update_utxo_13042023.down() -end +function migrate() + -- migration code from 2.4.0 to 2.4.3 + box.once("planetmint:v2.4.3", function() + box.space.utxos:drop() + utxos = box.schema.create_space('utxos', { if_not_exists = true }) + utxos:format({ + { name = 'id', type = 'string' }, + { name = 'amount' , type = 'unsigned' }, + { name = 'public_keys', type = 'array' }, + { name = 'condition', type = 'map' }, + { name = 'output_index', type = 'number' }, + { name = 'transaction_id' , type = 'string' } + }) + utxos:create_index('id', { + if_not_exists = true, + parts = {{ field = 'id', type = 'string' }} + }) + utxos:create_index('utxos_by_transaction_id', { + if_not_exists = true, + unique = false, + parts = {{ field = 'transaction_id', type = 'string' }} + }) + utxos:create_index('utxo_by_transaction_id_and_output_index', { + if_not_exists = true, + parts = { + { field = 'transaction_id', type = 'string' }, + { field = 'output_index', type = 'unsigned' } + } + }) + utxos:create_index('public_keys', { + if_not_exists = true, + unique = false, + parts = {{field = 'public_keys[*]', type = 'string' }} + }) + + atomic(1000, outputs:pairs(), function(output) + utxos:insert{output[0], output[1], output[2], output[3], output[4], output[5]} + end) + atomic(1000, utxos:pairs(), function(utxo) + spending_transaction = transactions.index.spending_transaction_by_id_and_output_index:select{utxo[5], utxo[4]} + if table.getn(spending_transaction) > 0 then + utxos:delete(utxo[0]) + end + end) + end) +end \ No newline at end of file diff --git a/planetmint/backend/tarantool/opt/migrations.lua b/planetmint/backend/tarantool/opt/migrations.lua deleted file mode 100644 index ca99736..0000000 --- a/planetmint/backend/tarantool/opt/migrations.lua +++ /dev/null @@ -1,79 +0,0 @@ -local functions = require('functions') -local migrations = {} - -migrations.update_utxo_13042023 = {} - -migrations.update_utxo_13042023.up = function() - if utxos.index.public_keys == nil then - box.space.utxos:drop() - utxos = box.schema.create_space('utxos', { if_not_exists = true }) - utxos:format({ - { name = 'id', type = 'string' }, - { name = 'amount' , type = 'unsigned' }, - { name = 'public_keys', type = 'array' }, - { name = 'condition', type = 'map' }, - { name = 'output_index', type = 'number' }, - { name = 'transaction_id' , type = 'string' } - }) - utxos:create_index('id', { - if_not_exists = true, - parts = {{ field = 'id', type = 'string' }} - }) - utxos:create_index('utxos_by_transaction_id', { - if_not_exists = true, - unique = false, - parts = {{ field = 'transaction_id', type = 'string' }} - }) - utxos:create_index('utxo_by_transaction_id_and_output_index', { - if_not_exists = true, - parts = { - { field = 'transaction_id', type = 'string' }, - { field = 'output_index', type = 'unsigned' } - } - }) - utxos:create_index('public_keys', { - if_not_exists = true, - unique = false, - parts = {{field = 'public_keys[*]', type = 'string' }} - }) - end - - outputs = box.space.outputs - functions.atomic(1000, outputs:pairs(), function(output) - utxos:insert{output[0], output[1], output[2], output[3], output[4], output[5]} - end) - functions.atomic(1000, utxos:pairs(), function(utxo) - spending_transaction = transactions.index.spending_transaction_by_id_and_output_index:select{utxo[5], utxo[4]} - if table.getn(spending_transaction) > 0 then - utxos:delete(utxo[0]) - end - end) -end - -migrations.update_utxo_13042023.down = function() - box.space.utxos:drop() - utxos = box.schema.create_space('utxos', { if_not_exists = true }) - utxos:format({ - { name = 'id', type = 'string' }, - { name = 'transaction_id', type = 'string' }, - { name = 'output_index', type = 'unsigned' }, - { name = 'utxo', type = 'map' } - }) - utxos:create_index('id', { - if_not_exists = true, - parts = {{ field = 'id', type = 'string' }} - }) - utxos:create_index('utxos_by_transaction_id', { - if_not_exists = true, - unique = false, - parts = {{ field = 'transaction_id', type = 'string' }} - }) - utxos:create_index('utxo_by_transaction_id_and_output_index', { - if_not_exists = true, - parts = { - { field = 'transaction_id', type = 'string' }, - { field = 'output_index', type = 'unsigned' } - }}) -end - -return migrations \ No newline at end of file diff --git a/planetmint/backend/tarantool/sync_io/schema.py b/planetmint/backend/tarantool/sync_io/schema.py index 3d86206..a51c23f 100644 --- a/planetmint/backend/tarantool/sync_io/schema.py +++ b/planetmint/backend/tarantool/sync_io/schema.py @@ -38,10 +38,5 @@ def create_tables(connection, dbname): @register_schema(TarantoolDBConnection) -def migrate_up(connection): - connection.connect().call("migrate_up") - - -@register_schema(TarantoolDBConnection) -def migrate_down(connection): - connection.connect().call("migrate_down") +def migrate(connection): + connection.connect().call("migrate") diff --git a/planetmint/commands/planetmint.py b/planetmint/commands/planetmint.py index 1dd441f..7532e01 100644 --- a/planetmint/commands/planetmint.py +++ b/planetmint/commands/planetmint.py @@ -259,13 +259,7 @@ def run_init(args): @configure_planetmint -def run_migrate_up(args): - validator = Validator() - schema.migrate_up(validator.models.connection) - - -@configure_planetmint -def run_migrate_down(args): +def run_migrate(args): validator = Validator() schema.migrate_up(validator.models.connection) @@ -377,8 +371,6 @@ def create_parser(): subparsers.add_parser("migrate_up", help="Migrate up") - subparsers.add_parser("migrate_down", help="Migrate down") - # parser for starting Planetmint start_parser = subparsers.add_parser("start", help="Start Planetmint") diff --git a/pyproject.toml b/pyproject.toml index 4f8ef4b..b1cc6f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "planetmint" -version = "2.4.2" +version = "2.4.3" description = "Planetmint: The Blockchain Database" authors = ["Planetmint contributors"] license = "AGPLv3"