12 files changed, 145 insertions(+), 85 deletions(-)

M .gitignore
M scripts/lua/entry.lua
M scripts/lua/migration/migration.lua
M scripts/lua/migration/test/test.lua
M scripts/lua/restore.lua
M scripts/lua/share/consts.lua
M scripts/lua/share/helpers.lua
D scripts/lua/test/device/user/temp/recovery_status.json
A scripts/lua/test/restore/restore1.tar
A scripts/lua/test/restore/restore2.tar
M scripts/lua/test/test.lua
M tools/init_databases.py

@@ 68,10 68,16 @@ test/test_env
 image/luts.bin
 image/assets/fonts/*
 
+scripts/lua/migration/test/databases/*
+scripts/lua/migration/test/migrations/*
 scripts/lua/test/assets*
-scripts/lua/test/update/user/temp/*
+scripts/lua/test/device/user/temp/recovery_status.json
+scripts/lua/test/update/user/*
 scripts/lua/test/update/target/*
 scripts/lua/test/update/system/*
-scripts/lua/test/update_udm/user/temp/*
+scripts/lua/test/update_udm/user/*
 scripts/lua/test/update_udm/target/*
 scripts/lua/test/update_udm/system/*
+scripts/lua/test/restore/user/*
+scripts/lua/test/restore/target/*
+scripts/lua/test/restore/system/*

@@ 37,7 37,7 @@ local function generate_report_file(boot_reason_str, success, message)
     local body = string.format(
         "{\"version\": \"%s\",\"branch\": \"%s\",\"revision\": \"%s\",\"operation\": \"%s\",\"successful\": %s,\"message\": \"%s\"}",
         rec.version(), rec.branch(), rec.revision(), boot_reason_str, tostring(success), message)
-    local fd = io.open(file_path, 'w')
+    local fd = assert(io.open(file_path, 'w'))
     fd:write(body)
     fd:close()
 end

@@ 37,10 37,10 @@ local function build_database_path(path, db_name)
 end
 
 local function db_exec(file, script, version)
-    local db = assert(sqlite3.open(file), string.format("file: %s", file))
-    assert(db:exec(script) == sqlite3.OK, string.format("script:\n%s\n", script))
+    local db = assert(sqlite3.open(file))
+    assert(db:exec(script) == sqlite3.OK, string.format("Script execution failed:\n%s\n", script))
     assert(db:exec(string.format("PRAGMA user_version=%u;", version)) == sqlite3.OK,
-        string.format("version: %d", version))
+        string.format("Setting database version: %d failed", version))
     db:close()
 end
 


@@ 90,26 90,33 @@ local function db_migrate_down(db_path, scripts, target_version)
 end
 
 local function print_db_set(db_set)
-    print("database set:")
+    print("Database set(name,version):")
     for name, version in pairs(db_set) do
-        print(string.format("'%s':%d",name,version))
+        print(string.format("   '%s':%d", name, version))
     end
 end
 
 local function validate_inputs(migration_dir, db_dir)
-    assert(helpers.exists(migration_dir), "Migrations directory does not exist")
+    assert(helpers.exists(migration_dir), "Migration directory does not exist")
     assert(helpers.exists(db_dir), "Databases directory does not exist")
     return true
 end
 
 local function migrate(db_path, scripts_up, scripts_down, target_version)
     local db_version = read_db_version(db_path)
+
     if db_version > target_version then
+        print(string.format("Performing migration-down of '%s' from version %d to %d", db_path, db_version,
+            target_version))
         return migration.down(db_path, scripts_down, target_version)
     end
     if db_version < target_version then
+        print(
+            string.format("Performing migration-up of '%s' from version %d to %d", db_path, db_version, target_version))
         return migration.up(db_path, scripts_up, target_version)
     end
+
+    print(string.format("Migration not needed, '%s' is already the newest version", db_path))
     return migration.retcode.OK
 end
 


@@ 148,8 155,8 @@ end
 -- @param db_set array of {<"database_name"> = <db_target_version>} entries
 -- @return @{retcode}
 function migration.migrate(db_dir, scripts_dir, db_set)
-    print(string.format("migrations scripts directory: '%s'", scripts_dir))
-    print(string.format("databases directory: '%s'", db_dir))
+    print(string.format("Migration scripts directory: '%s'", scripts_dir))
+    print(string.format("Databases directory: '%s'", db_dir))
     print_db_set(db_set)
 
     for name, version in pairs(db_set) do

@@ 1,6 1,5 @@
---Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved.
---For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md
-
+-- Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved.
+-- For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md
 package.path = package.path .. ";../?.lua;../../share/?.lua;../../share/?/?.lua"
 local lu = require("luaunit")
 local sqlite = require("lsqlite3complete")


@@ 41,35 40,35 @@ function test_migration_up_success()
     scripts[5] = "ALTER TABLE test_1 ADD new_column5 TEXT;"
 
     --- Migrate from version 0 to 1
-    spawn_db("test_1.sql", 0, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 1)
+    spawn_db("databases/test_1.sql", 0, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 1)
 
     -- Migrate from version 1 to 2
-    lu.assertEquals(migration.up("test_1.sql", scripts, 2), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 2)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 2), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 2)
 
     --- Migrate from version 0 to 5
-    spawn_db("test_1.sql", 0, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 5), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 5)
+    spawn_db("databases/test_1.sql", 0, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 5), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 5)
 end
 
 function test_migration_up_db_errors()
     scripts = {}
 
     --- Trigger DB error by trying to operate on empty database
-    spawn_db("test_1.sql", 1, "")
+    spawn_db("databases/test_1.sql", 1, "")
     scripts[1] = "ALTER TABLE test_1 ADD new_column TEXT;"
-    lu.assertError(migration.up, "test_1.sql", scripts, 2)
+    lu.assertError(migration.up, "databases/test_1.sql", scripts, 2)
 
     --- Target version set to the same value as the current DB version
-    spawn_db("test_1.sql", 1, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
+    spawn_db("databases/test_1.sql", 1, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
 
     --- Target version set to the lower number than the current DB version
-    spawn_db("test_1.sql", 2, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.WRONG_TARGET_VERSION)
+    spawn_db("databases/test_1.sql", 2, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.WRONG_TARGET_VERSION)
 end
 
 function test_migration_down_success()


@@ 81,35 80,35 @@ function test_migration_down_success()
     scripts[5] = "ALTER TABLE test_1 ADD new_column5 TEXT;"
 
     --- Migrate from version 2 to 1
-    spawn_db("test_1.sql", 2, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 1), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 1)
+    spawn_db("databases/test_1.sql", 2, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 1), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 1)
 
     -- Migrate from version 1 to 0
-    lu.assertEquals(migration.down("test_1.sql", scripts, 0), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 0)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 0), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 0)
 
     --- Migrate from version 5 to 0
-    spawn_db("test_1.sql", 5, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 0), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 0)
+    spawn_db("databases/test_1.sql", 5, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 0), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 0)
 end
 
 function test_migration_down_errors()
     scripts = {}
 
     --- Trigger DB error by trying to operate on empty database
-    spawn_db("test_1.sql", 2, "")
+    spawn_db("databases/test_1.sql", 2, "")
     scripts[1] = "ALTER TABLE test_1 ADD new_column TEXT;"
-    lu.assertError(migration.down, "test_1.sql", scripts, 1)
+    lu.assertError(migration.down, "databases/test_1.sql", scripts, 1)
 
     --- Target version set to the same value as the current DB version
-    spawn_db("test_1.sql", 1, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
+    spawn_db("databases/test_1.sql", 1, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
 
     --- Target version set to the higher number than the current DB version
-    spawn_db("test_1.sql", 2, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 3), migration.retcode.WRONG_TARGET_VERSION)
+    spawn_db("databases/test_1.sql", 2, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 3), migration.retcode.WRONG_TARGET_VERSION)
 end
 
 function test_automatic_migration()


@@ 125,6 124,13 @@ function test_automatic_migration()
     spawn_script("down.sql", "migrations", "test_2", 2, "delete from test_1 where _id=2;")
 
     migration.migrate("databases", "migrations", {
+        test_1 = 0,
+        test_2 = 0
+    })
+    lu.assertEquals(migration.get_db_version("databases/test_1.db"), 0)
+    lu.assertEquals(migration.get_db_version("databases/test_2.db"), 0)
+
+    migration.migrate("databases", "migrations", {
         test_1 = 2,
         test_2 = 2
     })

@@ 9,8 9,6 @@ local migration = require('migration')
 local restore = {}
 
 local unpacked_backup_dir = paths.temp_dir .. "/backup"
-local version_file = unpacked_backup_dir .. "/" .. consts.version_file
-local legacy_version_file = unpacked_backup_dir .. "/" .. consts.legacy_version_file
 
 restore.script_name = "restore.lua"
 restore.img_in_progress = "assets/gui_image_restore_in_progress.bin"


@@ 37,7 35,7 @@ local function unpack_backup()
     ltar.unpack(paths.backup_file, unpacked_backup_dir)
 end
 
-local function build_db_set(file)
+local function get_db_array_from_file(file)
     local contents = helpers.read_whole_file(file)
     local root = json.decode(contents)
     local set = {}


@@ 47,50 45,48 @@ local function build_db_set(file)
     return set
 end
 
-local function get_legacy_db_set()
-    local set = {
-        ["calllog"] = 0,
-        ["sms"] = 0,
-        ["events"] = 0,
-        ["settings_v2"] = 0,
-        ["notes"] = 0,
-        ["custom_quotes"] = 0,
-        ["predefined_quotes"] = 0,
-        ["contacts"] = 0,
-        ["alarms"] = 0,
-        ["notifications"] = 0,
-        ["multimedia"] = 0
-    }
+local function get_db_array_from_path(path)
+    local set = {}
+    for file in lfs.dir(path) do
+        local file_path = path .. "/" .. file
+        if file ~= "." and file ~= ".." then
+            if lfs.attributes(file_path, "mode") == "file" then
+                set[helpers.strip_from_extension(file)] = true;
+            end
+        end
+    end
+    return set
+end
+
+local function build_db_set()
+    local system_db_set = get_db_array_from_file(paths.version_file)
+    local backup_db_set = get_db_array_from_path(unpacked_backup_dir)
+    local set = {}
+    for name, version in pairs(system_db_set) do
+        if backup_db_set[name] then
+            set[name] = tonumber(version)
+        end
+    end
     return set
 end
 
 local function perform_db_migration()
     print("Performing database migration")
-    local dbset = {}
-    if helpers.exists(version_file) then
-        dbset = build_db_set(version_file)
-    else
-        assert(helpers.exists(legacy_version_file))
-        print("Legacy backup file, assuming legacy databases set")
-        dbset = get_legacy_db_set()
-    end
 
-    local result = migration.migrate(unpacked_backup_dir, paths.migration_scripts_dir, dbset)
+    local result = migration.migrate(unpacked_backup_dir, paths.migration_scripts_dir, build_db_set())
     assert(result == migration.retcode.OK, string.format("Database migration process failed with %d", result))
 end
 
 local function sync_databases()
     print("Syncing databases:")
-    print(string.format("Replacing old databases: '%s' with the ones from '%s'", paths.db_dir, unpacked_backup_dir))
-
-    helpers.rm_files_from_dir(paths.db_dir)
-    helpers.copy_dir(unpacked_backup_dir, paths.db_dir)
 
-    local version_file_path = paths.db_dir .. "/" .. consts.version_file
-    if not helpers.exists(version_file_path) then
-        version_file_path = paths.db_dir .. "/" .. consts.legacy_version_file
+    for name, _ in pairs(build_db_set()) do
+        local destination = paths.db_dir .. "/" .. name .. ".db"
+        local source = unpacked_backup_dir .. "/" .. name .. ".db"
+        print(string.format("Replacing '%s' with '%s'", destination, source));
+        assert(os.remove(destination))
+        helpers.copy_file(source, destination)
     end
-    assert(os.remove(version_file_path))
 end
 
 local function remove_cache()

@@ 1,7 1,12 @@
 local consts = {}
 
 consts.version_file = "version.json"
-consts.legacy_version_file = "backup.json" -- Pre-UDM backup package had version.json file named as backup.json
 consts.indexer_cache_file = ".directory_is_indexed"
 
+local match = {}
+
+-- Match only files with '.db' extensions and omit such files inside subdirectories
+match.only_db_files = '^[^%/]*%.db$'
+
+consts.match = match
 return consts

@@ 152,7 152,7 @@ function helpers.move_dir(from, where)
         if attr.mode == "directory" then
             assert(lfs.mkdir(build_path(where, name)))
         else
-            assert(os.rename(build_path(from, name),build_path(where, name)))
+            assert(os.rename(build_path(from, name), build_path(where, name)))
         end
     end
 end


@@ 235,6 235,14 @@ function helpers.get_file_extension(file)
     return file:match("^.+(%..+)$")
 end
 
+--- Strips file name from its extension
+-- @function strip_from_extension
+-- @param file file path
+-- @return file name without extension
+function helpers.strip_from_extension(file)
+    return file:match("^(.*)%..*$")
+end
+
 --- Create directory and all required subdirectories
 -- @function mkdirp
 -- @param file file path

@@ 1,1 0,0 @@
-{"version": "0.0.0","branch": "test","revision": "BABEF00D","operation": "restore","successful": false,"message": "../restore.lua:26: Not enough free space on user disk"}>
\ No newline at end of file

@@ 133,12 133,6 @@ describe("Factory/backup/restore scripts", function()
         recovery.sys.boot_reason_str.returns("restore")
         assert.has_no.error(invoke_entry)
     end)
-    it("invoke restore script, no free space", function()
-        recovery.sys.free_space.returns(10)
-        recovery.sys.boot_reason.returns(recovery.sys.boot_reason_codes.restore)
-        recovery.sys.boot_reason_str.returns("restore")
-        assert.has_no.error(invoke_entry)
-    end)
 end)
 
 local function remove_test_package(path)


@@ 151,6 145,43 @@ local function extract_test_package(path, where)
     os.execute(string.format("tar xf %s -C %s", path, where))
 end
 
+describe("Restore script", function()
+    recovery.sys.free_space.returns(1024 * 1024 * 1024)
+    recovery.sys.boot_reason.returns(recovery.sys.boot_reason_codes.restore)
+    recovery.sys.boot_reason_str.returns("restore")
+    recovery.sys.source_slot.returns("restore/system")
+    recovery.sys.target_slot.returns("restore/target")
+    recovery.sys.user.returns("restore/user")
+
+    package.loaded['paths'] = false
+    package.loaded['restore'] = false
+
+    it("the same set of databases", function()
+        -- Prepare test directory and its data
+        remove_test_package("restore/system")
+        remove_test_package("restore/user")
+        extract_test_package("restore/restore1.tar", "restore")
+        assert.has_no.error(require('restore').execute)
+    end)
+
+    it("legacy backup", function()
+        -- Prepare test directory and its data
+        remove_test_package("restore/system")
+        remove_test_package("restore/user")
+        extract_test_package("restore/restore2.tar", "restore")
+        assert.has_no.error(require('restore').execute)
+    end)
+
+    it("not enough disk space", function()
+        recovery.sys.free_space.returns(10)
+        -- Prepare test directory and its data
+        remove_test_package("restore/system")
+        remove_test_package("restore/user")
+        extract_test_package("restore/restore1.tar", "restore")
+        assert.has.error(require('restore').execute)
+    end)
+end)
+
 describe("Update script", function()
     recovery.sys.boot_reason.returns(recovery.sys.boot_reason_codes.update)
     recovery.sys.boot_reason_str.returns("update")

@@ 10,6 10,7 @@ import logging
 import sys
 import json
 import shutil
+from pathlib import Path
 
 log = logging.getLogger(__name__)
 logging.basicConfig(format='%(asctime)s [%(levelname)s]: %(message)s', level=logging.INFO)


@@ 25,6 26,7 @@ def migrate_database_up(database: str, migration_path: os.path, dst_directory: o
 
     db_name_full = f"{database}.db"
     dst_db_path = os.path.join(dst_directory, db_name_full)
+    Path(dst_db_path).unlink(missing_ok=True)
 
     ret = 0
     try: