upgrade_oniro.py

# SPDX-FileCopyrightText: Huawei Inc.
#
# SPDX-License-Identifier: Apache-2.0
# 
# A tool to upgrade to a newer version of Oniro

# Steps to upgrade:
# 1. Get the input from the user:
#   1.1. Target version
#   1.2. Build directory, can be empty or not and this will define some additional checks to perform
#   1.3. Machine and target to build, in order to test the upgraded build we need to know these things
#   1.4. Optional: Conf directory path, this can be specified in order to use a specific config directory.
#                   In case we have a clean build directory and this is not specified the default config will be used.
#   1.5. Optional: Flavour. In case the conf directory is not specified and the build directory is empty it becomes mandatory
#   1.6. Optional Flag: --dry-run: this enables the upgrade tool to create a local copy of the project and perform the upgrade on it.
#   1.7. Some other stuff to control the output of the tool (TBD)
#
# 2. Check for tool updates. If the newest version of Oniro has a new version of this script it should be upgraded before
#       Oniro's upgrade, then it should be run instead of the old one.
#
# 3. Pre-upgrade checks:
#   3.1. Check if "Target version" is newer than the current
#   3.2. Check if all the layers in the sample of Oniro are checked out in the correct state.
#   3.4. If the build directory is non-empty check if bblayers.conf have some additional layers, if positive flag them for the config update.
#   3.5. Flag all the .bbappends for update
#   3.6 Check if the flavor is correct if the conf dir is specified
#
# 4. Pre-upgrade backup to ensure that if something goes wrong we can restore a good state of the build. 
#       Option to also backup the build (tmp dir excluded) if the build directory is not empty.
#
# 5. Update:
#   5.1. Sync the repos
#   5.2. Update the .bbappends files (needs user confirmation) if needed
#   5.2. Run the script to init the environment
#   5.3. If specified copy the conf directory
#   5.4. Update the configs
#   5.5. If present delete the tmp directory
#   5.6. Build the target
# 6. In case of error let the user know that they can restore the old version using this tool
# 7. In case of a successful upgrade print/dump some additional information about the upgrade (CVE diff, buildhistory, ...)
#
# TODO think about a way to let the user know BEFORE the upgrade what changed so (s)he can choose. (maybe you can use git diff --compact-summary)
# TODO check if the script is executed in the top level directory of the project
# TODO check that if build_directory exists then it must be a directory

import argparse
import pathlib
import tarfile
import os
import shutil

script_description = "[WIP] A tool to upgrade to a newer version of Oniro"

# Script options
command = ""

# Upgrade options
target_version = ""
build_directory = None
machine = ""
flavour = ""
image = ""
conf_directory = None
dry_run = False
store_backup = False

# Backup options
verbose_backup = False

# Restore options
backup_archive_arg = ""

# Global vars
backup_archive_name = ""

def init_script_options():
    parser = argparse.ArgumentParser(description=script_description)
    subparsers = parser.add_subparsers(dest="command")

    # Upgrade subcommand
    upgrade_parser = subparsers.add_parser("upgrade", help="Upgrade Oniro.")
    # TODO a choice could be added in the version by checking all the tags of Oniro
    upgrade_parser.add_argument("target-version", type=str,
                        help="Specify the target version of Oniro you want to upgrade to")
    upgrade_parser.add_argument("build-directory", type=pathlib.Path,
                        help="Path to the build directory you want to use (can be non-existent)")
    # TODO a choice could be added in the machine
    upgrade_parser.add_argument("-m", "--machine", type=str, default="qemux86-64",
                        help="The target machine for the Oniro build. (qemux86-64 by default)")
    # TODO the option gathering can be automated by looking at oniro/flavours/
    upgrade_parser.add_argument("-f", "--flavour", type=str, choices=["linux", "zephyr", "freertos"], default="linux",
                        help="The flavor for the Oniro build. (linux by default)")
    # TODO the option gathering can be automated by looking at oniro/meta-oniro-core/recipes-core/images
    upgrade_parser.add_argument("-i", "--image", type=str, default="oniro-image-base",
                        choices=["oniro-image-base", "oniro-image-base-tests", "oniro-image-extra", "oniro-image-extra-tests", "zephyr-philosophers"],
                        help = "The image that will be built to check if the upgrade is successful (oniro-image-base by default)")
    upgrade_parser.add_argument("-c", "--conf-directory", type=pathlib.Path,
                        help="Path to the config directory you want to use. If omitted the default configs will be use.")
    upgrade_parser.add_argument("--dry-run", action="store_true",
                        help="Perform the upgrade on a local copy of the project.")
    upgrade_parser.add_argument("--store-backup", action="store_true",
                        help="Store the backup even if the upgrade was successful.")
    
    # TODO Backup subcommand
    restore_parser = subparsers.add_parser("backup", help="Backup Oniro manually.")
    restore_parser.add_argument("-v", "--verbose", action="store_true",
                        help="Activate verbose output mode")

    # Restore subcommand
    restore_parser = subparsers.add_parser("restore", help="Restore a local backup.")
    restore_parser.add_argument("backup_archive_arg", type=str,
                                help="Specify the path of a local backup archive to restore. Use 'latest' to use the latest backup found.")
    
    parsed_args = vars(parser.parse_args())
    
    global command
    command = parsed_args["command"]

    if command == "upgrade":
        global target_version, build_directory, machine, flavour, image, conf_directory, dry_run, store_backup
        target_version = parsed_args["target-version"]
        build_directory = parsed_args["build-directory"]
        machine = parsed_args["machine"]
        flavour = parsed_args["flavour"]
        image = parsed_args["image"]
        conf_directory = parsed_args["conf_directory"]
        dry_run = parsed_args["dry_run"]
        store_backup = parsed_args["store_backup"]
    elif command == "backup":
        global verbose_backup
        verbose_backup = parsed_args["verbose"]
    elif command == "restore":
        global backup_archive_arg
        backup_archive_arg = parsed_args["backup_archive_arg"]

def upgrade_tool():
    # TODO
    return

def perform_pre_upgrade_checks():
    # TODO
    return

def backup_oniro():
    # This will back up:
    #   - All layers specified in the Oniro Manifest
    #   - .repo folder
    #   - If build_directory is not empty, backup its content excluding tmp* and tmp-newlib* directories
    # TODO add function description
    # TODO add an option to backup build directories / DV layers ?
    from upgrade_oniro_utils import get_manifest_projects
    from datetime import datetime
    
    backup_folders = [layer["path"] for layer in get_manifest_projects()] # add all manifest layers
    backup_folders.append(".repo")

    if build_directory and build_directory.exists():
        for child in  build_directory.iterdir():
            if child.is_dir() and "tmp" not in child.name and child.name != "downloads":
                backup_folders.append(f"{build_directory.name}/{child.name}")
    
    # Create a metadata file containing all the folders that have been backed up
    with open("backup_folders", "w") as f:
        for folder in backup_folders:
            f.write(f"{folder}\n")

    print("Oniro backup started")
    archive_name = f"upgrade_oniro_backup-{datetime.timestamp(datetime.now())}.tar.gz"
    with tarfile.open(archive_name, "w:gz") as tar_archive:
        tar_archive.add("backup_folders")
        for folder in backup_folders:
            tar_archive.add(folder)
            if verbose_backup:
                print(f"{folder} has been backed up")
    
    # Remove the uncompressed copy of backup_folders
    os.remove("backup_folders")

    print(f"Backup created at {archive_name}")

    return

def restore_oniro():
    # TODO add function description
    # TODO add an exception handling if the metadata file is not found
    from upgrade_oniro_utils import find_latest_backup

    if backup_archive_arg == "latest":
        backup_archive_name = find_latest_backup()
        if backup_archive_name is None:
            print("No backup found, aborting...")
            return
    elif os.path.exists(backup_archive_arg):
        # TODO add a check to be sure that the selected path is actually a Oniro backup
        backup_archive_name = backup_archive_arg
    else:
        print(f"'{backup_archive_arg}' has not been found, aborting...")
        return

    print(f"Restoring {backup_archive_name}")
    with tarfile.open(backup_archive_name) as backup_archive:
        # Extract the metadata file and delete all the old folders
        backup_folders_file = backup_archive.extract("backup_folders")
        with open("backup_folders") as bf:
            for folder in bf.readlines():
                shutil.rmtree(folder.strip())
        # Restore the backed up folders
        backup_archive.extractall(".")

    # Remove the uncompressed copy of backup_folders
    os.remove("backup_folders")

    print(f"{backup_archive_name} successfully restored!")
    return

def upgrade_oniro():
    # TODO implement a verbose output and a silent one
    # TODO check all the returns codes

    import subprocess
    from upgrade_oniro_utils import update_bblayers, get_layers

    # Get old layers for later use (used to update bblayers.conf)
    old_layers, _ = get_layers(pathlib.Path(f"oniro/flavours/{flavour}/bblayers.conf.sample"))

    # Update oniro repo
    if target_version == "kirkstone":
        oniro_checkout = subprocess.Popen(f'cd oniro; git checkout eclipse/kirkstone; git pull eclipse kirkstone', shell=True, stdout=subprocess.PIPE)
    else:
        oniro_checkout = subprocess.Popen(f'cd oniro; git checkout {target_version}', shell=True, stdout=subprocess.PIPE)
    print(oniro_checkout.stdout.read().decode("utf-8"))

    # Sync the repos
    repo_sync = subprocess.Popen(r'repo sync -m $(pwd)/oniro/manifests/default.xml', shell=True, stdout=subprocess.PIPE)
    print(repo_sync.stdout.read().decode("utf-8"))
    
    # Update the .bbappends files (needs user confirmation) if needed
    # TODO
    
    # Run the script to init the environment
    init_env = subprocess.Popen(f'bash -c "TEMPLATECONF=../oniro/flavours/{flavour} source ./oe-core/oe-init-build-env {build_directory}"', shell=True, stdout=subprocess.PIPE)

    # If specified copy the conf directory
    if conf_directory and conf_directory.is_dir():
        print(f"Coping custom conf directory into {build_directory}...", end="")
        shutil.rmtree(build_directory / "conf")
        shutil.copytree(conf_directory, build_directory / "conf", symlinks = True)
        print("Done")
    
        # Update the configs
        print("Updating bblayers...", end="")
        update_bblayers(build_directory, flavour, old_layers)
        print("Done")
        # TODO update local.conf? or just use the default in case something changed in the sample?
    
    # If present delete the tmp directory
    tmp_folder = "tmp-newlib" if flavour == "zephyr" else "tmp"
    build_tmp_dir = pathlib.Path(build_directory / tmp_folder)
    if build_tmp_dir.exists():
        print(f"Removing {build_tmp_dir}")
        shutil.rmtree(build_tmp_dir)
    
    # Build the target
    build_oniro = subprocess.run(
         f'bash -c "TEMPLATECONF=../oniro/flavours/{flavour} source ./oe-core/oe-init-build-env {build_directory}; MACHINE={machine} bitbake {image}"',
         shell=True)

    return

if __name__ == "__main__":
    init_script_options()
    upgrade_tool()

    if command == "upgrade":
        perform_pre_upgrade_checks()
        backup_oniro()
        # restore_oniro()
        upgrade_oniro()
    
    elif command == "backup":
        backup_oniro()
    
    elif command == "restore":
        restore_oniro()