diff --git a/aidge_onnx/onnx_import.py b/aidge_onnx/onnx_import.py
index 0ac16ba059ced52702ab9d6e0e615c6652e6322e..a8de870d0df2d0ad82a4113f54fdafa141061cf3 100644
--- a/aidge_onnx/onnx_import.py
+++ b/aidge_onnx/onnx_import.py
@@ -15,7 +15,7 @@ import colorama
from onnx import numpy_helper
import onnx
from .node_import import ONNX_NODE_CONVERTER_, generic
-from .utils import *
+from .utils import onnx_to_aidge_model_names
def load_onnx(filename: str, verbose: bool = False):
"""Load an ONNX file and convert it into a :py:class:`aidge_core.GraphView`.
@@ -107,7 +107,7 @@ def _load_onnx2graphview(model:onnx.ModelProto, verbose:bool = False):
# Clean model if some issues in the model
# might affect Aidge in the next steps
- model = clean2aidge(model)
+ model = onnx_to_aidge_model_names(model)
if verbose : print(f"\nGetting Initializers\n====================")
# Get the initializers
for i in model.graph.initializer:
diff --git a/aidge_onnx/unit_tests/compare_layers_aidge_onnx.py b/aidge_onnx/unit_tests/compare_layers_aidge_onnx.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a312b82088b642156c26ff6e6d2932338c2fca9
--- /dev/null
+++ b/aidge_onnx/unit_tests/compare_layers_aidge_onnx.py
@@ -0,0 +1,283 @@
+import aidge_core
+import aidge_backend_cpu
+import aidge_onnx
+from aidge_onnx.utils import onnx_to_aidge_name
+
+import argparse
+from datetime import datetime
+
+import matplotlib.pyplot as plt
+
+import onnx
+import onnxruntime
+
+import numpy as np
+import numpy.typing as npt
+import os
+
+
+def set_all_model_nodes_as_output(
+ model: onnx.ModelProto, initial_output_nb: int
+) -> onnx.ModelProto:
+ """
+ Retrieves a given onnx model and tags all its nodes as outputs
+ args :
+ model : Loaded onnx model.
+ initial_output_nb : Number of outputs of the model. Allows to rebuild
+ the outputs in the right order.
+ """
+ shape_info = onnx.shape_inference.infer_shapes(model)
+ # 1. retrieve output layers, they will be added at the very end of the process
+ output_layers = np.array(
+ [model.graph.output.pop() for node in range(initial_output_nb)]
+ )
+ output_layers = np.flip(output_layers)
+
+ # 2. tag all nodes as outputs
+ nodes_to_set = []
+ for idx, node in enumerate(shape_info.graph.value_info):
+ nodes_to_set.append(node)
+
+ model.graph.output.extend(
+ nodes_to_set
+ ) # in inference stage, these tensor will be added to output dict.
+
+ # 3. re-add output_layers back to its original place at the end ouf output
+ model.graph.output.extend(output_layers)
+
+ return model
+
+
+def create_network_inputs(
+ session: onnxruntime.InferenceSession,
+) -> dict[str, npt.NDArray]:
+ """
+ From an inference session, generate random inputs for the model
+ args :
+ session : loaded onnx model.
+ returns :
+ Dictionnary containing tagged inputs.
+ """
+ inputs = {}
+ for input_val in session.get_inputs():
+ input_data = (
+ np.random.random(size=input_val.shape).astype(np.float32) - 0.5
+ ) * 2 # generating data in [-1 , 1 ]
+ # input_data = np.ones(input_val.shape).astype(np.float32)
+ inputs[input_val.name] = input_data
+ return inputs
+
+
+def find_node_with_name(
+ aidge_nodes: set[aidge_core.Node], name: str
+) -> aidge_core.Node:
+ """
+ Find node in a set of nodes by checking its name
+ """
+ for node in aidge_nodes:
+ if node.name() == name:
+ return node
+ return -1
+
+
+def compute_error(
+ estimated_output: npt.NDArray,
+ groundtruth_output: npt.NDArray,
+ relative_precision=1e-3,
+ absolute_presision=1e-4,
+):
+ """
+ Computes the error and an error threshold,
+ based on a groundtruth (onnx output) and an estimation (aidge output).
+ This function will automatically try to flatten layers whose shape does not
+ match because the remove flatten is automatically applied on the network.
+ If after flattening the sizes still doesn't match, the program will exit
+ with error code -1
+ args :
+ estimated_output : aidge node output
+ groundtruth_output : onnx node output
+ return :
+ (error, maximum_threshold)
+
+ """
+ error = np.abs(estimated_output - groundtruth_output)
+ err_threshold = relative_precision * np.abs(groundtruth_output) + absolute_presision
+ if (
+ estimated_output.shape != groundtruth_output.shape
+ ): # try to flatten the output bc if size differ it means that removeFlatten recipe has been applied
+ print("Warn: shape of output differ even trying to flatten them to match.")
+ print(f"aidge output shape : {estimated_output.shape}")
+ print(f"onnx output shape : {estimated_output.shape}")
+
+ estimated_output = estimated_output.flatten()
+ groundtruth_output = groundtruth_output.flatten()
+ error = np.abs(estimated_output - groundtruth_output) / np.max(
+ groundtruth_output
+ )
+ err_threshold = (
+ relative_precision * np.abs(groundtruth_output) + absolute_presision
+ )
+ if estimated_output.shape != groundtruth_output.shape:
+ print(
+ "ERROR : shape of output differ even after trying to flatten them, aborting."
+ )
+ print(f"aidge output shape : {estimated_output.shape}")
+ print(f"onnx output shape : {groundtruth_output.shape}")
+ exit(-1)
+ return (error, err_threshold)
+
+
+def compare_layers_aidge_onnx(
+ model_path: str, verbose=False, display=False, save_graph=False
+):
+ """
+ This function is the main of the script.
+ It will :
+ 1. Modify the onnx model to tag all its nodes as ouputs.
+ 2. Save it and load it as onnxruntime and aidge model (and delete temp file)
+ 3. Infer with both models.
+ 4. Check line per line the outputs of the model.
+ 5. If verbose is True print the nodes that failed.
+ 6. If enabled, Display/save the graph of mean(error)/mean(threshold) ratio.
+ args :
+ model_path : path to onnx model.
+ verbose : print the nodes whose size do not match
+ display : outputs a graph of the ratio error/threshold
+ save_graph : if true saves the graph to a given name
+ """
+
+ onnx_model: onnx.ModelProto = onnx.load(model_path)
+ # creating session to get its output nb
+ onnx_session = onnxruntime.InferenceSession(model_path)
+ nb_outputs = len(onnx_session.get_outputs())
+ onnx_model = set_all_model_nodes_as_output(onnx_model, nb_outputs)
+ shape_info = onnx.shape_inference.infer_shapes(onnx_model)
+ onnx.save(shape_info, f"{model_path}_shape_info.onnx")
+
+ modified_model_fpath = f"{model_path}_all_layers_are_outputs.onnx"
+ onnx.save_model(onnx_model, modified_model_fpath)
+
+ # load modified models and delete temp file
+ onnx_session = onnxruntime.InferenceSession(modified_model_fpath)
+ aidge_model = aidge_onnx.load_onnx(modified_model_fpath)
+ os.remove(modified_model_fpath)
+
+ # prepare input and outputs
+ inputs = create_network_inputs(onnx_session)
+
+ # onnx inference
+ onnx_node_names = [output.name for output in onnx_session.get_outputs()]
+ onnx_outputs = onnx_session.run(onnx_node_names, inputs)
+
+ # preparing inputs for aidge
+ for idx, (input_name, input_val) in enumerate(inputs.items()):
+ input_tensor = aidge_core.Tensor(np.array(input_val))
+ input_node = aidge_core.Producer(input_tensor, f"{input_name}")
+ input_node.add_child(
+ aidge_model, 0, [sorted(aidge_model.get_input_nodes())[idx], 0]
+ )
+ aidge_model.add(input_node)
+
+ aidge_model.save("post_add_child")
+ # remove invalid layers
+ aidge_core.remove_flatten(aidge_model)
+
+ aidge_model.compile("cpu", aidge_core.DataType.Float32)
+
+ # aidge inference
+ scheduler = aidge_core.SequentialScheduler(aidge_model)
+ scheduler.forward(False, False)
+
+ # Build the list of nodes as it is on onnx
+ aidge_node_names = [onnx_to_aidge_name(name) for name in onnx_node_names]
+ aidge_nodes = [
+ node for node in aidge_model.get_nodes() if node.name() in aidge_node_names
+ ]
+
+ # Iterate over all nodes and compare their outputs
+ all_layers_identical = True
+ layers_error_above_threshold = []
+ layers_error = []
+ for idx, name in enumerate(onnx_node_names):
+ if name.find("Flatten") != -1: # flatten layers have been removed
+ print(f"Skipping layer : {name}")
+ continue
+ aidge_node_to_search = onnx_to_aidge_name(name)
+ aidge_node = find_node_with_name(aidge_nodes, aidge_node_to_search)
+ if aidge_node == -1:
+ print(
+ f"ERROR : no aidge node found with name {aidge_node_to_search}"
+ "\nLeaving output comparison early.",
+ )
+ break
+ onnx_node_output = onnx_outputs[idx]
+ aidge_node_output = np.array(aidge_node.get_operator().get_output(0))
+
+ (error, err_threshold) = compute_error(aidge_node_output, onnx_node_output)
+ if display:
+ layers_error.append(
+ np.mean(error) / np.mean(err_threshold)
+ if np.mean(err_threshold) != 0
+ else 0
+ )
+ if np.any(error[error > err_threshold]): # error tolerance :
+ all_layers_identical = False
+ layers_error_above_threshold.append(
+ (idx, name, np.mean(error), np.mean(err_threshold))
+ )
+
+ # create graph to save or display
+ fig = plt.figure()
+ plt.plot(layers_error)
+ if save_graph:
+ now = datetime.now()
+ dt_string = now.strftime("%Y-%m-%d_%H-%M-%S")
+ fig.savefig(f"error_threshold_ratio_{dt_string}.jpg")
+ if display:
+ plt.show()
+
+ if all_layers_identical:
+ if verbose:
+ print("Aidge outputs the same result as ONNXRT")
+ return True
+ else:
+ if verbose:
+ print("AIDGE and ONNX outputs differ at the following nodes :")
+ print("IDX\tNAME\t\tMax relative error in the layer\t\t4.Error threshold")
+ print(*layers_error_above_threshold, sep="\n")
+ return False
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ prog="Compare Layers",
+ description="This program is meant to compare, side-by-side, for each layer, an onnx model and its imported aidge model"
+ "the output of each layer."
+ '\nIt will print the layers that differ "a lot" between ONNX and AIDGE models'
+ "(more informations on that in the compute_error function.\n"
+ "If specified, this script will show / save the graph of the mean(errors) / mean(threshold).",
+ epilog="WARNING : This script for now only supports nodes with a single output.",
+ )
+ parser.add_argument(
+ "--onnx-model",
+ "-m",
+ help="File path of onnx model",
+ required=True,
+ action="store",
+ )
+ parser.add_argument(
+ "--display",
+ "-d",
+ help="Displays the generated graph",
+ action="store_true",
+ )
+ parser.add_argument(
+ "--save-graph",
+ "-s",
+ help="Saves the graph in jpg",
+ action="store_true",
+ )
+ args = parser.parse_args()
+ compare_layers_aidge_onnx(
+ args.onnx_model, verbose=True, display=args.display, save_graph=args.save_graph
+ )
diff --git a/aidge_onnx/utils.py b/aidge_onnx/utils.py
index 4129919982c545625d564d14ec0689118b4242b9..7a5029bae421a059dba59ce4306d7db86ca9ee6a 100644
--- a/aidge_onnx/utils.py
+++ b/aidge_onnx/utils.py
@@ -18,49 +18,35 @@ _MAP_NP_ONNX_TYPE = {
}
_MAP_ONNX_NP_TYPE = {v: k for k, v in _MAP_NP_ONNX_TYPE.items()}
-def replace_with_underscore(input_string):
- # Use the replace method to replace '/' with '_'
- modified_string = input_string.replace('/', '_')
- modified_string = modified_string.replace('.', '_')
- return modified_string
-def replace_digit_first_character(input_string):
- modified_string = input_string
- if len(input_string) > 0 and input_string[0].isdigit():
- modified_string = "data_" + input_string
- return modified_string
-
-def replace(name):
- # All replacements to do in the name
- new_name = replace_with_underscore(name)
- new_name = replace_digit_first_character(new_name)
- return new_name
-
-def clean_names(model: onnx.ModelProto):
+def onnx_to_aidge_model_names(model: onnx.ModelProto):
+ """
+ Change the name of each node of the model from onnx convention to aidge's one
+ args :
+ model : to modify
+ return :
+ model : modified
+ """
for i in model.graph.initializer:
- i.name = replace(i.name)
+ i.name = onnx_to_aidge_name(i.name)
for n in model.graph.node:
if len(n.name) > 0 and n.name[0].isdigit():
new_name = "layer_" + n.name
n.name = new_name
for index, i in enumerate(n.input):
- n.input[index] = replace(i)
+ n.input[index] = onnx_to_aidge_name(i)
for index, o in enumerate(n.output):
- n.output[index] = replace(o)
+ n.output[index] = onnx_to_aidge_name(o)
for i in model.graph.input:
- i.name = replace(i.name)
+ i.name = onnx_to_aidge_name(i.name)
for o in model.graph.output:
- o.name = replace(o.name)
+ o.name = onnx_to_aidge_name(o.name)
return model
-def clean2aidge(model:onnx.ModelProto):
- model = clean_names(model)
- return model
-
def numpy_to_onnx_type(np_dtype):
if np_dtype not in _MAP_NP_ONNX_TYPE:
raise ValueError(f"Unsupported NumPy dtype: {np_dtype}")
@@ -72,3 +58,11 @@ def onnx_to_numpy_type(onnx_type):
raise ValueError(f"Unsupported ONNX TensorProto type: {onnx_type}")
np_dtype = _MAP_ONNX_NP_TYPE[onnx_type]
return np_dtype
+
+def onnx_to_aidge_name(name: str) -> str:
+ """
+ Translates onnx node naming convention to aidge naming convention
+ """
+ name = name.replace("/", "_").replace(".", "_")
+ name = name if (len(name) == 0 or not name[0].isdigit()) else "data_" + name
+ return name