diff --git a/CHANGELOG.md b/CHANGELOG.md index 332e106..213f209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ For reference, the possible headings are: * **Known Issues** * **Notes** +## [2.4.2] - 2023-13-04 +* **Added** planetmint migration commands + ## [2.4.1] - 2023-11-04 * **Removed** Fastquery class * **Changed** UTXO space updated to resemble outputs diff --git a/docker-compose.yml b/docker-compose.yml index f3daf35..6bfa5ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,9 @@ services: - "3303:3303" - "8081:8081" volumes: - - ./planetmint/backend/tarantool/init.lua:/opt/tarantool/init.lua + - ./planetmint/backend/tarantool/opt/init.lua:/opt/tarantool/init.lua + - ./planetmint/backend/tarantool/opt/functions.lua:/opt/tarantool/functions.lua + - ./planetmint/backend/tarantool/opt/migrations.lua:/opt/tarantool/migrations.lua entrypoint: tarantool /opt/tarantool/init.lua restart: always planetmint: diff --git a/planetmint/backend/schema.py b/planetmint/backend/schema.py index e3eceef..2af754f 100644 --- a/planetmint/backend/schema.py +++ b/planetmint/backend/schema.py @@ -137,6 +137,30 @@ def init_database(connection, dbname): raise NotImplementedError +@singledispatch +def migrate_up(connection): + """Migrate database up + + Args: + connection (:class:`~planetmint.backend.connection.Connection`): an + existing connection to use to migrate the database. + Creates one if not given. + """ + raise NotImplementedError + + +@singledispatch +def migrate_down(connection): + """Migrate database down + + Args: + connection (:class:`~planetmint.backend.connection.Connection`): an + existing connection to use to migrate the database. + Creates one if not given. + """ + raise NotImplementedError + + def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. diff --git a/planetmint/backend/tarantool/opt/functions.lua b/planetmint/backend/tarantool/opt/functions.lua new file mode 100644 index 0000000..24a2d5c --- /dev/null +++ b/planetmint/backend/tarantool/opt/functions.lua @@ -0,0 +1,20 @@ +local fiber = require('fiber') + +local export = {} + +function export.atomic(batch_size, iter, fn) + box.atomic(function() + local i = 0 + for _, x in iter:unwrap() do + fn(x) + i = i + 1 + if i % batch_size == 0 then + box.commit() + fiber.yield() -- for read-only operations when `commit` doesn't yield + box.begin() + end + end + end) +end + +return export \ No newline at end of file diff --git a/planetmint/backend/tarantool/init.lua b/planetmint/backend/tarantool/opt/init.lua similarity index 97% rename from planetmint/backend/tarantool/init.lua rename to planetmint/backend/tarantool/opt/init.lua index 83832cc..7dac85d 100644 --- a/planetmint/backend/tarantool/init.lua +++ b/planetmint/backend/tarantool/opt/init.lua @@ -1,3 +1,5 @@ +local migrations = require('migrations') + box.cfg{listen = 3303} box.once("bootstrap", function() @@ -331,3 +333,13 @@ end function delete_output( id ) box.space.outputs:delete(id) end + +function migrate_up() + migrations.update_utxo_13042023.up() + -- add newer migrations below +end + +function migrate_down() + -- add newer migrations above + migrations.update_utxo_13042023.down() +end diff --git a/planetmint/backend/tarantool/opt/migrations.lua b/planetmint/backend/tarantool/opt/migrations.lua new file mode 100644 index 0000000..ca99736 --- /dev/null +++ b/planetmint/backend/tarantool/opt/migrations.lua @@ -0,0 +1,79 @@ +local functions = require('functions') +local migrations = {} + +migrations.update_utxo_13042023 = {} + +migrations.update_utxo_13042023.up = function() + if utxos.index.public_keys == nil then + box.space.utxos:drop() + utxos = box.schema.create_space('utxos', { if_not_exists = true }) + utxos:format({ + { name = 'id', type = 'string' }, + { name = 'amount' , type = 'unsigned' }, + { name = 'public_keys', type = 'array' }, + { name = 'condition', type = 'map' }, + { name = 'output_index', type = 'number' }, + { name = 'transaction_id' , type = 'string' } + }) + utxos:create_index('id', { + if_not_exists = true, + parts = {{ field = 'id', type = 'string' }} + }) + utxos:create_index('utxos_by_transaction_id', { + if_not_exists = true, + unique = false, + parts = {{ field = 'transaction_id', type = 'string' }} + }) + utxos:create_index('utxo_by_transaction_id_and_output_index', { + if_not_exists = true, + parts = { + { field = 'transaction_id', type = 'string' }, + { field = 'output_index', type = 'unsigned' } + } + }) + utxos:create_index('public_keys', { + if_not_exists = true, + unique = false, + parts = {{field = 'public_keys[*]', type = 'string' }} + }) + end + + outputs = box.space.outputs + functions.atomic(1000, outputs:pairs(), function(output) + utxos:insert{output[0], output[1], output[2], output[3], output[4], output[5]} + end) + functions.atomic(1000, utxos:pairs(), function(utxo) + spending_transaction = transactions.index.spending_transaction_by_id_and_output_index:select{utxo[5], utxo[4]} + if table.getn(spending_transaction) > 0 then + utxos:delete(utxo[0]) + end + end) +end + +migrations.update_utxo_13042023.down = function() + box.space.utxos:drop() + utxos = box.schema.create_space('utxos', { if_not_exists = true }) + utxos:format({ + { name = 'id', type = 'string' }, + { name = 'transaction_id', type = 'string' }, + { name = 'output_index', type = 'unsigned' }, + { name = 'utxo', type = 'map' } + }) + utxos:create_index('id', { + if_not_exists = true, + parts = {{ field = 'id', type = 'string' }} + }) + utxos:create_index('utxos_by_transaction_id', { + if_not_exists = true, + unique = false, + parts = {{ field = 'transaction_id', type = 'string' }} + }) + utxos:create_index('utxo_by_transaction_id_and_output_index', { + if_not_exists = true, + parts = { + { field = 'transaction_id', type = 'string' }, + { field = 'output_index', type = 'unsigned' } + }}) +end + +return migrations \ No newline at end of file diff --git a/planetmint/backend/tarantool/sync_io/schema.py b/planetmint/backend/tarantool/sync_io/schema.py index 83cf319..3d86206 100644 --- a/planetmint/backend/tarantool/sync_io/schema.py +++ b/planetmint/backend/tarantool/sync_io/schema.py @@ -35,3 +35,13 @@ def create_database(connection, dbname): @register_schema(TarantoolDBConnection) def create_tables(connection, dbname): connection.connect().call("init") + + +@register_schema(TarantoolDBConnection) +def migrate_up(connection): + connection.connect().call("migrate_up") + + +@register_schema(TarantoolDBConnection) +def migrate_down(connection): + connection.connect().call("migrate_down") diff --git a/planetmint/commands/planetmint.py b/planetmint/commands/planetmint.py index 6a9bc35..1dd441f 100644 --- a/planetmint/commands/planetmint.py +++ b/planetmint/commands/planetmint.py @@ -258,6 +258,18 @@ def run_init(args): _run_init() +@configure_planetmint +def run_migrate_up(args): + validator = Validator() + schema.migrate_up(validator.models.connection) + + +@configure_planetmint +def run_migrate_down(args): + validator = Validator() + schema.migrate_up(validator.models.connection) + + @configure_planetmint def run_drop(args): """Drop the database""" @@ -363,6 +375,10 @@ def create_parser(): subparsers.add_parser("drop", help="Drop the database") + subparsers.add_parser("migrate_up", help="Migrate up") + + subparsers.add_parser("migrate_down", help="Migrate down") + # parser for starting Planetmint start_parser = subparsers.add_parser("start", help="Start Planetmint") diff --git a/pyproject.toml b/pyproject.toml index e20f8a8..4f8ef4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "planetmint" -version = "2.4.1" +version = "2.4.2" description = "Planetmint: The Blockchain Database" authors = ["Planetmint contributors"] license = "AGPLv3"