scripts: add script to list duplicate recipes

Signed-off-by: Davide Gardenal davide.gardenal@huawei.com

scripts/list_duplicate_recipes.py

+import subprocess
+
+# This script to generates a list of duplicate recipes in Oniro.
+# The folder you are running this will determin the scope of the search.
+# To get a global list use the root project directory
+# Author: Davide Gardenal <davide.gardenl@huawei.com>
+
+# Get all the .bb files and generate a list
+# Command explained
+# find $(find * -maxdepth 0 | sed "/build-.*/d" | sed "/^bitbake.*/d") -> list all the files in the current directory
+# excluding all the builds and bitbake (no duplicate recipes in there)
+# grep -E "\.bb$" -> select only the .bb files
+# sed "s:\(.*\)/\([^/]*\)\.bb$:\2 \1:" -> spit the .bb file name with its path
+# sort -> sort by recipe name
+# sed "s:^\(.*\)\_\(.*\) \(.*\)$:\1 \3 \2:" -> extract the version number (output is PN path PV)
+bb_files_report_p = subprocess.Popen(r'find $(find * -maxdepth 0 | sed "/build-.*/d" | sed "/^bitbake.*/d") | \
+                                                 grep -E "\.bb$" | \
+                                                     sed "s:\(.*\)/\([^/]*\)\.bb$:\2 \1:" | \
+                                                         sort | \
+                                                             sed "s:^\(.*\)\_\(.*\) \(.*\)$:\1 \3 \2:"'
+                                                             , shell=True, stdout=subprocess.PIPE)
+
+bb_files_content = bb_files_report_p.stdout.read().decode("utf-8")
+
+bb_files_dicts = []
+
+# This iters for each line except for the last that is empty
+for line in bb_files_content.split("\n")[:-1]:
+    l_tokens = line.split(" ")
+    # extract the layer
+    layer_path_tokens = l_tokens[1].split("/")
+    r_layer = [x for x in layer_path_tokens if "meta" in x][-1:][0]
+
+    # special case for generate-zephyr-machines (meta is not the layer)
+    if "generate-zephyr-machines" ==  l_tokens[0]:
+        r_layer = "meta-zephyr-bsp"
+    # special case for distro-feed-configs (meta is not the layer)
+    if "distro-feed-configs" ==  l_tokens[0]:
+        r_layer = "meta-oe"
+    # special case for meta-python, is not a layer
+    if "meta-oe/dynamic-layers/meta-python" in l_tokens[1]:
+        r_layer = "meta-oe"
+    if "meta-intel/dynamic-layers/meta-python" in l_tokens[1]:
+        r_layer = "meta-intel"
+    if "meta-raspberrypi/dynamic-layers/meta-python" in l_tokens[1]:
+        r_layer = "meta-raspberrypi"
+
+    # Check if the version is known
+    r_version = "unknown"
+    if len(l_tokens) > 2:
+        r_version = l_tokens[2]
+
+    bb_files_dicts.append({"name": l_tokens[0], "layer": r_layer, "version": r_version})
+
+
+# Join the dictionaries so that we have {"name": name, "info": (layer, version)}
+# The recipes are sorted so we can pass once and create the new list
+
+bb_joined_dicts = [{"name": bb_files_dicts[0]["name"], "info": [(bb_files_dicts[0]["layer"], bb_files_dicts[0]["version"])]}]
+
+for i in range(1, len(bb_files_dicts)):
+    name = bb_files_dicts[i]["name"]
+    if name == bb_joined_dicts[len(bb_joined_dicts)-1]["name"]:
+        # It's the same recipe then add the new info
+        new_info = (bb_files_dicts[i]["layer"], bb_files_dicts[i]["version"])
+        bb_joined_dicts[len(bb_joined_dicts)-1]["info"].append(new_info)
+    else:
+        # It's another recipe then create a new entry in the list
+        new_recipe = {
+            "name": bb_files_dicts[i]["name"], 
+            "info": [(bb_files_dicts[i]["layer"], bb_files_dicts[i]["version"])]
+            }
+        bb_joined_dicts.append(new_recipe)
+
+# Check if the layers are different, True if so, False otherwise 
+def are_layers_different(infos):
+    first_layer, _ = infos[0]
+    for current_layer, _ in infos:
+        if first_layer != current_layer:
+            return True
+    return False
+
+# This doesn't check if the version is duplicate but only checks if in the version is
+# present "imx" or "arc" or nothing more then once
+# For example if version_layer_1 is "1.0.0imx" and version_layer_2 is "2.4.3" this does
+# not count as a duplicate, "1.0.0imx" and "2.4.3imx" does. Same for "arc"
+def is_version_type_duplicate(infos):
+    found = {"imx": 0, "arc": 0, "none": 0}
+    for _, version in infos:
+        if "imx" in version: found["imx"] += 1
+        elif "arc" in version: found["arc"] += 1
+        else: found["none"] += 1
+
+    return any([item > 1 for item in found.values()])
+
+
+# Select and print the duplicate recipes with the corresponding layers and versions
+for entry in bb_joined_dicts:
+    # consider only if the recipe is contained in more that one layer
+    to_print = len(entry["info"]) > 1
+    # check if there is at least one layer that is different from the others
+    to_print = to_print and are_layers_different(entry["info"])
+    # check the version for "imx" o "arc"
+    to_print = to_print and is_version_type_duplicate(entry["info"])
+
+    if to_print:
+        out_line = f"{entry['name']}"
+        for i in entry["info"]:
+            layer, version = i
+            out_line += f" {layer}/{version}"
+        print(out_line)

[Marta Rybczynska]

determin -> determine

[Marta Rybczynska]

I would go for it at the end only, I think... This is at least what we see right now