fetch_configure_sources.py

#!/usr/bin/env python3

# SPDX-License-Identifier: GPL-2.0-only
# SPDX-FileCopyrightText: Alberto Pianon <pianon@array.eu>


import os
import re
import json
import subprocess

from bb.tinfoil import Tinfoil

def bash(command, cwd=None):
    out = subprocess.run(
        command,
        shell=True,
        executable="/bin/bash",
        cwd=cwd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    stdout = out.stdout.decode()
    stderr = out.stderr.decode()
    if out.returncode != 0:
        raise Exception(
            f"Command '{command}' failed. Output is:\n" f"{stdout}\n{stderr}"
        )
    return stdout, stderr

def parse_pkgdata(pkgdata_file):
    pkgdata_text = pkgdata_file.read()
    pkgdata = {}
    p = re.compile('^([^:]+): (.*)$')
    lines = pkgdata_text.split('\n')
    for line in lines:
        m = p.match(line)
        if m:
            key = m.group(1)
            value = json.loads(m.group(2)) if key == 'FILES_INFO' else m.group(2).strip()
            pkgdata.update({ key: value })
    return pkgdata


def main():
    """Simple function to retrieve all recipes for the packages included 
    in the current manifest(s), check if they have do_configure (or at least
    do_patch or do_unpack) task, and run the task. 
    It is intended to be run after a bitbake quick build that leverages SSTATE 
    cache (and thus does not fetch nor configure sources, because it reuses 
    existing build artifacts), to fetch and configure sources (but without
    rebuilding them) in order to be able to match binary files with source files.
    The tasks are run only on recipes that correspond to packages that are actually
    included in the final images: bitbake will automatic run all the other
    necessary tasks on build dependency recipes
    """
    with Tinfoil() as tf:

        tf.prepare()

        image_dir = tf.config_data.getVar('DEPLOY_DIR_IMAGE')
        machine = tf.config_data.getVar('MACHINE')
        pkgdata_dir = tf.config_data.getVar('PKGDATA_DIR')
        
        manifests = [
            f for f in os.listdir(image_dir)
            if f.endswith('.manifest')
            and os.path.islink(os.path.join(image_dir,f))
        ]

        package_names = []
        for manifest in manifests:
            image = os.path.basename(manifest).replace(f'-{machine}.manifest', '')
            print(f"parsing image '{image}'")
            with open(os.path.join(image_dir, manifest), 'r') as f:
                package_names += [ line.split()[0] for line in f if line ]
        package_names = sorted(list(set(package_names)))
        if '' in package_names:
            package_names.remove('')

        print("retrieving recipe for each package")
        recipes = {}
        for package_name in package_names:
            pkgdata_file = os.path.join(pkgdata_dir, 'runtime-reverse', package_name)
            with open(pkgdata_file, 'r') as f:
                pkgdata = parse_pkgdata(f)
            recipe = pkgdata.get('PN')
            if recipe not in recipes:
                print(f"parsing recipe {recipe}")
                r = tf.parse_recipe(recipe)
                workdir = r.getVar("WORKDIR")
                recipes[recipe] = workdir

    print("running listtasks...")
    bash(f'bitbake -c listtasks {" ".join(recipes)}')

    print("retrieving recipes with tasks to run")
    
    tasks = ["do_configure", "do_patch", "do_unpack"]
    task_recipes = { task: [] for task in tasks }

    for recipe, workdir in recipes.items():
        listtasks_logfile = os.path.join(workdir, "temp", "log.do_listtasks")
        with open(listtasks_logfile) as f:
            listtasks_log = f.read()
        for task in task_recipes:
            if task in listtasks_log:
                task_recipes[task].append(recipe)
                print(f"recipe {recipe} has task {task}")
                break
    
    for task, recipes in task_recipes.items():
        if not recipes:
            continue
        print(f"running {task} on all possible recipes/targets (it may take some time)")
        bash(f'bitbake -c {task} {" ".join(recipes)}')
    
    print("DONE!")
    

if __name__ == "__main__":
    main()