Merge branch 'feat/script_compare_layers_aidge_onnx' into 'dev'

feat/script_compare_layers_aidge_onnx

See merge request eclipse/aidge/aidge_onnx!33

aidge_onnx/onnx_import.py

@@ -15,7 +15,7 @@ import colorama
 from onnx import numpy_helper
 import onnx
 from .node_import import ONNX_NODE_CONVERTER_, generic
-from .utils import *
+from .utils import onnx_to_aidge_model_names
 
 def load_onnx(filename: str, verbose: bool = False):
     """Load an ONNX file and convert it into a :py:class:`aidge_core.GraphView`.
@@ -107,7 +107,7 @@ def _load_onnx2graphview(model:onnx.ModelProto, verbose:bool = False):
 
     # Clean model if some issues in the model
     # might affect Aidge in the next steps
-    model = clean2aidge(model)
+    model = onnx_to_aidge_model_names(model)
     if verbose : print(f"\nGetting Initializers\n====================")
     # Get the initializers
     for i in model.graph.initializer:

aidge_onnx/unit_tests/compare_layers_aidge_onnx.py

+import aidge_core
+import aidge_backend_cpu
+import aidge_onnx
+from aidge_onnx.utils import onnx_to_aidge_name
+
+import argparse
+from datetime import datetime
+
+import matplotlib.pyplot as plt
+
+import onnx
+import onnxruntime
+
+import numpy as np
+import numpy.typing as npt
+import os
+
+
+def set_all_model_nodes_as_output(
+    model: onnx.ModelProto, initial_output_nb: int
+) -> onnx.ModelProto:
+    """
+    Retrieves a given onnx model and tags all its nodes as outputs
+    args :
+    model : Loaded onnx model.
+    initial_output_nb : Number of outputs of the model. Allows to rebuild
+                        the outputs in the right order.
+    """
+    shape_info = onnx.shape_inference.infer_shapes(model)
+    # 1. retrieve output layers, they will be added at the very end of the process
+    output_layers = np.array(
+        [model.graph.output.pop() for node in range(initial_output_nb)]
+    )
+    output_layers = np.flip(output_layers)
+
+    # 2. tag all nodes as outputs
+    nodes_to_set = []
+    for idx, node in enumerate(shape_info.graph.value_info):
+        nodes_to_set.append(node)
+
+    model.graph.output.extend(
+        nodes_to_set
+    )  #  in inference stage, these tensor will be added to output dict.
+
+    # 3. re-add output_layers back to its original place at the end ouf output
+    model.graph.output.extend(output_layers)
+
+    return model
+
+
+def create_network_inputs(
+    session: onnxruntime.InferenceSession,
+) -> dict[str, npt.NDArray]:
+    """
+    From an inference session, generate random inputs for the model
+    args :
+        session : loaded onnx model.
+    returns :
+        Dictionnary containing tagged inputs.
+    """
+    inputs = {}
+    for input_val in session.get_inputs():
+        input_data = (
+            np.random.random(size=input_val.shape).astype(np.float32) - 0.5
+        ) * 2  # generating data in [-1 , 1 ]
+        # input_data = np.ones(input_val.shape).astype(np.float32)
+        inputs[input_val.name] = input_data
+    return inputs
+
+
+def find_node_with_name(
+    aidge_nodes: set[aidge_core.Node], name: str
+) -> aidge_core.Node:
+    """
+    Find node in a set of nodes by checking its name
+    """
+    for node in aidge_nodes:
+        if node.name() == name:
+            return node
+    return -1
+
+
+def compute_error(
+    estimated_output: npt.NDArray,
+    groundtruth_output: npt.NDArray,
+    relative_precision=1e-3,
+    absolute_presision=1e-4,
+):
+    """
+    Computes the error and an error threshold,
+    based on a groundtruth (onnx output) and an estimation (aidge output).
+    This function will automatically try to flatten layers whose shape does not
+    match because the remove flatten is automatically applied on the network.
+    If after flattening the sizes still doesn't match, the program will exit
+    with error code -1
+    args :
+        estimated_output : aidge node output
+        groundtruth_output : onnx node output
+    return :
+        (error, maximum_threshold)
+
+    """
+    error = np.abs(estimated_output - groundtruth_output)
+    err_threshold = relative_precision * np.abs(groundtruth_output) + absolute_presision
+    if (
+        estimated_output.shape != groundtruth_output.shape
+    ):  # try to flatten the output bc if size differ it means that removeFlatten recipe has been applied
+        print("Warn: shape of output differ even trying to flatten them to match.")
+        print(f"aidge output shape : {estimated_output.shape}")
+        print(f"onnx output shape : {estimated_output.shape}")
+
+        estimated_output = estimated_output.flatten()
+        groundtruth_output = groundtruth_output.flatten()
+        error = np.abs(estimated_output - groundtruth_output) / np.max(
+            groundtruth_output
+        )
+        err_threshold = (
+            relative_precision * np.abs(groundtruth_output) + absolute_presision
+        )
+        if estimated_output.shape != groundtruth_output.shape:
+            print(
+                "ERROR : shape of output differ even after trying to flatten them, aborting."
+            )
+            print(f"aidge output shape : {estimated_output.shape}")
+            print(f"onnx output shape : {groundtruth_output.shape}")
+            exit(-1)
+    return (error, err_threshold)
+
+
+def compare_layers_aidge_onnx(
+    model_path: str, verbose=False, display=False, save_graph=False
+):
+    """
+    This function is the main of the script.
+    It will :
+    1. Modify the onnx model to tag all its nodes as ouputs.
+    2. Save it and load it as onnxruntime and aidge model (and delete temp file)
+    3. Infer with both models.
+    4. Check line per line the outputs of the model.
+    5. If verbose is True print the nodes that failed.
+    6. If enabled, Display/save the graph of mean(error)/mean(threshold) ratio.
+    args :
+        model_path : path to onnx model.
+        verbose : print the nodes whose size do not match
+        display : outputs a graph of the ratio error/threshold
+        save_graph : if true saves the graph to a given name
+    """
+
+    onnx_model: onnx.ModelProto = onnx.load(model_path)
+    # creating session to get its output nb
+    onnx_session = onnxruntime.InferenceSession(model_path)
+    nb_outputs = len(onnx_session.get_outputs())
+    onnx_model = set_all_model_nodes_as_output(onnx_model, nb_outputs)
+    shape_info = onnx.shape_inference.infer_shapes(onnx_model)
+    onnx.save(shape_info, f"{model_path}_shape_info.onnx")
+
+    modified_model_fpath = f"{model_path}_all_layers_are_outputs.onnx"
+    onnx.save_model(onnx_model, modified_model_fpath)
+
+    # load modified models and delete temp file
+    onnx_session = onnxruntime.InferenceSession(modified_model_fpath)
+    aidge_model = aidge_onnx.load_onnx(modified_model_fpath)
+    os.remove(modified_model_fpath)
+
+    # prepare input and outputs
+    inputs = create_network_inputs(onnx_session)
+
+    # onnx inference
+    onnx_node_names = [output.name for output in onnx_session.get_outputs()]
+    onnx_outputs = onnx_session.run(onnx_node_names, inputs)
+
+    # preparing inputs for aidge
+    for idx, (input_name, input_val) in enumerate(inputs.items()):
+        input_tensor = aidge_core.Tensor(np.array(input_val))
+        input_node = aidge_core.Producer(input_tensor, f"{input_name}")
+        input_node.add_child(
+            aidge_model, 0, [sorted(aidge_model.get_input_nodes())[idx], 0]
+        )
+        aidge_model.add(input_node)
+
+    aidge_model.save("post_add_child")
+    # remove invalid layers
+    aidge_core.remove_flatten(aidge_model)
+
+    aidge_model.compile("cpu", aidge_core.DataType.Float32)
+
+    # aidge inference
+    scheduler = aidge_core.SequentialScheduler(aidge_model)
+    scheduler.forward(False, False)
+
+    # Build the list of nodes as it is on onnx
+    aidge_node_names = [onnx_to_aidge_name(name) for name in onnx_node_names]
+    aidge_nodes = [
+        node for node in aidge_model.get_nodes() if node.name() in aidge_node_names
+    ]
+
+    # Iterate over all nodes and compare their outputs
+    all_layers_identical = True
+    layers_error_above_threshold = []
+    layers_error = []
+    for idx, name in enumerate(onnx_node_names):
+        if name.find("Flatten") != -1:  # flatten layers have been removed
+            print(f"Skipping layer : {name}")
+            continue
+        aidge_node_to_search = onnx_to_aidge_name(name)
+        aidge_node = find_node_with_name(aidge_nodes, aidge_node_to_search)
+        if aidge_node == -1:
+            print(
+                f"ERROR : no aidge node found with name {aidge_node_to_search}"
+                "\nLeaving output comparison early.",
+            )
+            break
+        onnx_node_output = onnx_outputs[idx]
+        aidge_node_output = np.array(aidge_node.get_operator().get_output(0))
+
+        (error, err_threshold) = compute_error(aidge_node_output, onnx_node_output)
+        if display:
+            layers_error.append(
+                np.mean(error) / np.mean(err_threshold)
+                if np.mean(err_threshold) != 0
+                else 0
+            )
+        if np.any(error[error > err_threshold]):  # error tolerance :
+            all_layers_identical = False
+            layers_error_above_threshold.append(
+                (idx, name, np.mean(error), np.mean(err_threshold))
+            )
+
+    # create graph to save or display
+    fig = plt.figure()
+    plt.plot(layers_error)
+    if save_graph:
+        now = datetime.now()
+        dt_string = now.strftime("%Y-%m-%d_%H-%M-%S")
+        fig.savefig(f"error_threshold_ratio_{dt_string}.jpg")
+    if display:
+        plt.show()
+
+    if all_layers_identical:
+        if verbose:
+            print("Aidge outputs the same result as ONNXRT")
+        return True
+    else:
+        if verbose:
+            print("AIDGE and ONNX outputs differ at the following nodes :")
+            print("IDX\tNAME\t\tMax relative error in the layer\t\t4.Error threshold")
+            print(*layers_error_above_threshold, sep="\n")
+        return False
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Compare Layers",
+        description="This program is meant to compare, side-by-side, for each layer, an onnx model and its imported aidge model"
+        "the output of each layer."
+        '\nIt will print the layers that differ "a lot" between ONNX and AIDGE models'
+        "(more informations on that in the compute_error function.\n"
+        "If specified, this script will show / save the graph of the mean(errors) / mean(threshold).",
+        epilog="WARNING : This script for now only supports nodes with a single output.",
+    )
+    parser.add_argument(
+        "--onnx-model",
+        "-m",
+        help="File path of onnx model",
+        required=True,
+        action="store",
+    )
+    parser.add_argument(
+        "--display",
+        "-d",
+        help="Displays the generated graph",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--save-graph",
+        "-s",
+        help="Saves the graph in jpg",
+        action="store_true",
+    )
+    args = parser.parse_args()
+    compare_layers_aidge_onnx(
+        args.onnx_model, verbose=True, display=args.display, save_graph=args.save_graph
+    )

aidge_onnx/utils.py

@@ -18,49 +18,35 @@ _MAP_NP_ONNX_TYPE = {
     }
 _MAP_ONNX_NP_TYPE = {v: k for k, v in _MAP_NP_ONNX_TYPE.items()}
 
-def replace_with_underscore(input_string):
-    # Use the replace method to replace '/' with '_'
-    modified_string = input_string.replace('/', '_')
-    modified_string = modified_string.replace('.', '_')
-    return modified_string
 
-def replace_digit_first_character(input_string):
-    modified_string = input_string
-    if len(input_string) > 0 and input_string[0].isdigit():
-        modified_string = "data_" + input_string
-    return modified_string
-
-def replace(name):
-    # All replacements to do in the name
-    new_name = replace_with_underscore(name)
-    new_name = replace_digit_first_character(new_name)
-    return new_name
-
-def clean_names(model: onnx.ModelProto):
+def onnx_to_aidge_model_names(model: onnx.ModelProto):
+    """
+    Change the name of each node of the model from onnx convention to aidge's one
+    args :
+        model : to modify
+    return :
+        model : modified
+    """
     for i in model.graph.initializer:
-        i.name = replace(i.name)
+        i.name = onnx_to_aidge_name(i.name)
 
     for n in model.graph.node:
         if len(n.name) > 0 and n.name[0].isdigit():
             new_name = "layer_" + n.name
             n.name = new_name
         for index, i in enumerate(n.input):
-            n.input[index] = replace(i)
+            n.input[index] = onnx_to_aidge_name(i)
         for index, o in enumerate(n.output):
-            n.output[index] = replace(o)
+            n.output[index] = onnx_to_aidge_name(o)
 
     for i in model.graph.input:
-        i.name = replace(i.name)
+        i.name = onnx_to_aidge_name(i.name)
 
     for o in model.graph.output:
-        o.name = replace(o.name)
+        o.name = onnx_to_aidge_name(o.name)
 
     return model
 
-def clean2aidge(model:onnx.ModelProto):
-    model = clean_names(model)
-    return model
-
 def numpy_to_onnx_type(np_dtype):
     if np_dtype not in _MAP_NP_ONNX_TYPE:
         raise ValueError(f"Unsupported NumPy dtype: {np_dtype}")
@@ -72,3 +58,11 @@ def onnx_to_numpy_type(onnx_type):
         raise ValueError(f"Unsupported ONNX TensorProto type: {onnx_type}")
     np_dtype = _MAP_ONNX_NP_TYPE[onnx_type]
     return np_dtype
+
+def onnx_to_aidge_name(name: str) -> str:
+    """
+    Translates onnx node naming convention to aidge naming convention
+    """
+    name = name.replace("/", "_").replace(".", "_")
+    name = name if (len(name) == 0 or not name[0].isdigit()) else "data_" + name
+    return name