upgrade_oniro: expand backup functionalities

Now when performing a backup you can specify extra folders
or files to backup (in addition to the defaults).
Now all the folders starting with "meta-" will be backed up.
The restore tool has been updated to support the above mentioned
improvements.

Signed-off-by: Davide Gardenal <davide.gardenal@huawei.com>

scripts/upgrade_oniro/upgrade_oniro.py

+#!/usr/bin/python
+
 # SPDX-FileCopyrightText: Huawei Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -44,6 +46,7 @@
 # TODO check that if build_directory exists then it must be a directory
 
 import argparse
+from importlib.resources import path
 import pathlib
 import tarfile
 import os
@@ -66,6 +69,7 @@ store_backup = False
 
 # Backup options
 verbose_backup = False
+extra_folders = []
 
 # Restore options
 backup_archive_arg = ""
@@ -101,10 +105,13 @@ def init_script_options():
     upgrade_parser.add_argument("--store-backup", action="store_true",
                         help="Store the backup even if the upgrade was successful.")
     
-    # TODO Backup subcommand
-    restore_parser = subparsers.add_parser("backup", help="Backup Oniro manually.")
-    restore_parser.add_argument("-v", "--verbose", action="store_true",
+    
+    # Backup subcommand
+    backup_parser = subparsers.add_parser("backup", help="Backup Oniro manually.")
+    backup_parser.add_argument("-v", "--verbose", action="store_true",
                         help="Activate verbose output mode")
+    backup_parser.add_argument("-ef","--extra-folders", action="extend", nargs="+", type=pathlib.Path, 
+                        help = "Specify a list of extra folders to backup")
 
     # Restore subcommand
     restore_parser = subparsers.add_parser("restore", help="Restore a local backup.")
@@ -127,9 +134,10 @@ def init_script_options():
         dry_run = parsed_args["dry_run"]
         store_backup = parsed_args["store_backup"]
     elif command == "backup":
-        global verbose_backup
+        global verbose_backup, extra_folders
         verbose_backup = parsed_args["verbose"]
-    elif command == "restore":
+        extra_folders = parsed_args["extra_folders"]
+    elif command == "restore":  
         global backup_archive_arg
         backup_archive_arg = parsed_args["backup_archive_arg"]
 
@@ -151,21 +159,39 @@ def backup_oniro():
     from upgrade_oniro_utils import get_manifest_projects
     from datetime import datetime
     
-    backup_folders = [layer["path"] for layer in get_manifest_projects()] # add all manifest layers
+    # Save all the layers / repo in the manifest
+    backup_folders = [layer["path"] for layer in get_manifest_projects()]
+    # Save repo workspace
     backup_folders.append(".repo")
 
+    # I a build directory is specified save the sstates, the cache and conf
     if build_directory and build_directory.exists():
         for child in  build_directory.iterdir():
             if child.is_dir() and "tmp" not in child.name and child.name != "downloads":
                 backup_folders.append(f"{build_directory.name}/{child.name}")
     
+    # Save all the folders starting with "meta-"
+    this_dir = pathlib.Path(".")
+    for child in this_dir.iterdir():
+        dir_name = child.name
+        if len(dir_name) >4 and dir_name[:5] == "meta-":
+            backup_folders.append(dir_name)
+
+    # Save all the folders specified by the user
+    backup_folders += [f.name for f in extra_folders if f.exists()]
+
+    # To guarantee no duplicates
+    backup_folders = set(backup_folders)
+
     # Create a metadata file containing all the folders that have been backed up
     with open("backup_folders", "w") as f:
         for folder in backup_folders:
             f.write(f"{folder}\n")
 
     print("Oniro backup started")
-    archive_name = f"upgrade_oniro_backup-{datetime.timestamp(datetime.now())}.tar.gz"
+    if not os.path.exists("upgrade_oniro_backups"):
+        os.mkdir("upgrade_oniro_backups")
+    archive_name = f"upgrade_oniro_backups/upgrade_oniro_backup-{datetime.timestamp(datetime.now())}.tar.gz"
     with tarfile.open(archive_name, "w:gz") as tar_archive:
         tar_archive.add("backup_folders")
         for folder in backup_folders:
@@ -183,6 +209,8 @@ def backup_oniro():
 def restore_oniro():
     # TODO add function description
     # TODO add an exception handling if the metadata file is not found
+    # TODO if something created some files / directories we are not able to 
+    # delete those and we may end up with a inconsistent state
     from upgrade_oniro_utils import find_latest_backup
 
     if backup_archive_arg == "latest":
@@ -203,7 +231,8 @@ def restore_oniro():
         backup_folders_file = backup_archive.extract("backup_folders")
         with open("backup_folders") as bf:
             for folder in bf.readlines():
-                shutil.rmtree(folder.strip())
+                if os.path.exists(folder):
+                    shutil.rmtree(folder.strip())
         # Restore the backed up folders
         backup_archive.extractall(".")
 

scripts/upgrade_oniro/upgrade_oniro_utils.py

@@ -20,16 +20,19 @@ def get_manifest_projects():
     root = tree.getroot()
     return [project.attrib for project in root.findall("project")]
 
-def find_latest_backup():
+def find_latest_backup(base_dir):
     """
     Find the latest backup of Oniro. This will search only in the
     current directory.
 
+    Args:
+        base_dir: folder where all the backups are stored
+
     Returns:
         The latest backup name if found, None otherwise.
     """
     
-    this_dir = Path(".")
+    this_dir = Path(base_dir)
     backup_info_list = []
     for item in this_dir.iterdir():
         if "upgrade_oniro_backup" in item.name:
@@ -40,7 +43,7 @@ def find_latest_backup():
             }
             backup_info_list.append(backup_info)
     
-    return None if backup_info_list == [] else max(backup_info_list, key=lambda x: int(f'{x["timestamp"][0]}{x["timestamp"][1]}'))["name"]
+    return None if backup_info_list == [] else "%s/%s" % (base_dir, max(backup_info_list, key=lambda x: int(f'{x["timestamp"][0]}{x["timestamp"][1]}'))["name"])
 
 def get_layers(bbl_file):
     # TODO add description