Add utxo migration (#379)

* added migration script for utxo space
* added migration commands
* changelog and version bump
* added db call to command

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

CHANGELOG.md

@@ -25,6 +25,9 @@ For reference, the possible headings are:
 * **Known Issues**
 * **Notes**
 
+## [2.4.2] - 2023-13-04
+* **Added** planetmint migration commands
+
 ## [2.4.1] - 2023-11-04
 * **Removed** Fastquery class
 * **Changed** UTXO space updated to resemble outputs

docker-compose.yml

@@ -22,7 +22,9 @@ services:
       - "3303:3303"
       - "8081:8081"
     volumes:
-      - ./planetmint/backend/tarantool/init.lua:/opt/tarantool/init.lua
+      - ./planetmint/backend/tarantool/opt/init.lua:/opt/tarantool/init.lua
+      - ./planetmint/backend/tarantool/opt/functions.lua:/opt/tarantool/functions.lua
+      - ./planetmint/backend/tarantool/opt/migrations.lua:/opt/tarantool/migrations.lua
     entrypoint: tarantool /opt/tarantool/init.lua
     restart: always
   planetmint:

planetmint/backend/schema.py

@@ -137,6 +137,30 @@ def init_database(connection, dbname):
     raise NotImplementedError
 
 
+@singledispatch
+def migrate_up(connection):
+    """Migrate database up
+
+    Args:
+        connection (:class:`~planetmint.backend.connection.Connection`): an
+            existing connection to use to migrate the database.
+            Creates one if not given.
+    """
+    raise NotImplementedError
+
+
+@singledispatch
+def migrate_down(connection):
+    """Migrate database down
+
+    Args:
+        connection (:class:`~planetmint.backend.connection.Connection`): an
+            existing connection to use to migrate the database.
+            Creates one if not given.
+    """
+    raise NotImplementedError
+
+
 def validate_language_key(obj, key):
     """Validate all nested "language" key in `obj`.
 

planetmint/backend/tarantool/opt/functions.lua

@@ -0,0 +1,20 @@
+local fiber = require('fiber')
+
+local export = {}
+
+function export.atomic(batch_size, iter, fn)
+    box.atomic(function()
+        local i = 0
+        for _, x in iter:unwrap() do
+            fn(x)
+            i = i + 1
+            if i % batch_size == 0 then
+                box.commit()
+                fiber.yield() -- for read-only operations when `commit` doesn't yield
+                box.begin()
+            end
+        end
+    end)
+end
+
+return export

planetmint/backend/tarantool/opt/init.lua

@@ -1,3 +1,5 @@
+local migrations = require('migrations')
+
 box.cfg{listen = 3303}
 
 box.once("bootstrap", function()
@@ -331,3 +333,13 @@ end
 function delete_output( id )
     box.space.outputs:delete(id)
 end
+
+function migrate_up()
+    migrations.update_utxo_13042023.up()
+    -- add newer migrations below
+end
+
+function migrate_down()
+    -- add newer migrations above
+    migrations.update_utxo_13042023.down()
+end

planetmint/backend/tarantool/opt/migrations.lua

@@ -0,0 +1,79 @@
+local functions = require('functions')
+local migrations = {}
+
+migrations.update_utxo_13042023 = {}
+
+migrations.update_utxo_13042023.up = function()
+    if utxos.index.public_keys == nil then
+        box.space.utxos:drop()
+        utxos = box.schema.create_space('utxos', { if_not_exists = true })
+        utxos:format({
+            { name = 'id', type = 'string' },
+            { name = 'amount' , type = 'unsigned' },
+            { name = 'public_keys', type = 'array' },
+            { name = 'condition', type = 'map' },
+            { name = 'output_index', type = 'number' },
+            { name = 'transaction_id' , type = 'string' }
+        })
+        utxos:create_index('id', { 
+            if_not_exists = true,
+            parts = {{ field = 'id', type = 'string' }}
+        })
+        utxos:create_index('utxos_by_transaction_id', {
+            if_not_exists = true,
+            unique = false,
+            parts = {{ field = 'transaction_id', type = 'string' }}
+        })
+        utxos:create_index('utxo_by_transaction_id_and_output_index', { 
+            if_not_exists = true,
+            parts = {
+                { field = 'transaction_id', type = 'string' },
+                { field = 'output_index', type = 'unsigned' }
+            }
+        })
+        utxos:create_index('public_keys', { 
+            if_not_exists = true,
+            unique = false,
+            parts = {{field = 'public_keys[*]', type  = 'string' }}
+        })
+    end
+
+    outputs = box.space.outputs
+    functions.atomic(1000, outputs:pairs(), function(output)
+        utxos:insert{output[0], output[1], output[2], output[3], output[4], output[5]}
+    end)
+    functions.atomic(1000, utxos:pairs(), function(utxo) 
+        spending_transaction = transactions.index.spending_transaction_by_id_and_output_index:select{utxo[5], utxo[4]}
+        if table.getn(spending_transaction) > 0 then
+            utxos:delete(utxo[0])
+        end
+    end)
+end
+
+migrations.update_utxo_13042023.down = function()
+    box.space.utxos:drop()
+    utxos = box.schema.create_space('utxos', { if_not_exists = true })
+    utxos:format({
+        { name = 'id', type = 'string' },
+        { name = 'transaction_id', type = 'string' },
+        { name = 'output_index', type = 'unsigned' },
+        { name = 'utxo', type = 'map' }
+    })
+    utxos:create_index('id', { 
+        if_not_exists = true,
+        parts = {{ field = 'id', type = 'string' }}
+    })
+    utxos:create_index('utxos_by_transaction_id', {
+        if_not_exists = true,
+        unique = false,
+        parts = {{ field = 'transaction_id', type = 'string' }}
+    })
+    utxos:create_index('utxo_by_transaction_id_and_output_index', { 
+        if_not_exists = true,
+        parts = {
+            { field = 'transaction_id', type = 'string' },
+            { field = 'output_index', type = 'unsigned' }
+    }})
+end
+
+return migrations

planetmint/backend/tarantool/sync_io/schema.py

@@ -35,3 +35,13 @@ def create_database(connection, dbname):
 @register_schema(TarantoolDBConnection)
 def create_tables(connection, dbname):
     connection.connect().call("init")
+
+
+@register_schema(TarantoolDBConnection)
+def migrate_up(connection):
+    connection.connect().call("migrate_up")
+
+
+@register_schema(TarantoolDBConnection)
+def migrate_down(connection):
+    connection.connect().call("migrate_down")

planetmint/commands/planetmint.py

@@ -258,6 +258,18 @@ def run_init(args):
     _run_init()
 
 
+@configure_planetmint
+def run_migrate_up(args):
+    validator = Validator()
+    schema.migrate_up(validator.models.connection)
+
+
+@configure_planetmint
+def run_migrate_down(args):
+    validator = Validator()
+    schema.migrate_up(validator.models.connection)
+
+
 @configure_planetmint
 def run_drop(args):
     """Drop the database"""
@@ -363,6 +375,10 @@ def create_parser():
 
     subparsers.add_parser("drop", help="Drop the database")
 
+    subparsers.add_parser("migrate_up", help="Migrate up")
+
+    subparsers.add_parser("migrate_down", help="Migrate down")
+
     # parser for starting Planetmint
     start_parser = subparsers.add_parser("start", help="Start Planetmint")
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "planetmint"
-version = "2.4.1"
+version = "2.4.2"
 description = "Planetmint: The Blockchain Database"
 authors = ["Planetmint contributors"]
 license = "AGPLv3"