Issue with Conv operator and backend CUDA : giving bad results when the output is bigger than the input
Problem description
When doing inference on mobileNetV2.7 on Aidge with backend CUDA, some of the convolutions layers are giving results significantly different from the output using ONNX Runtime.
I tried to isolate the problem and I think it happens when the output of the convolution is bigger (in size) than the input.
For example conv_model_1.onnx:
the previous convolution gives exactly the same output on aidge backend_cuda as onnx runtime. However when I inverse the shapes and make the output bigger than the input, I get different results from onnx runtime conv_model_2.onnx:
and here is the code I use to compare to ONNX runtime:
DEVICE = "cuda"
import aidge_core
if DEVICE == "cuda":
import aidge_backend_cuda
else:
import aidge_backend_cpu
import aidge_onnx
import numpy as np
import onnxruntime
import time
model_path = "conv_model_1.onnx"
input = np.random.randn(1,96,112,112).astype(np.float32)
print("************* Aidge Inference ***************")
aidge_model = aidge_onnx.load_onnx(model_path)
input_tensor = aidge_core.Tensor(input)
input_node = aidge_core.Producer(input_tensor, "input")
input_node.get_operator().set_datatype(aidge_core.DataType.Float32)
input_node.get_operator().set_backend(DEVICE)
input_node.add_child(aidge_model)
aidge_model.add(input_node)
aidge_model.set_datatype(aidge_core.DataType.Float32)
aidge_model.set_backend(DEVICE)
scheduler = aidge_core.SequentialScheduler(aidge_model)
scheduler.forward()
aidge_out_arr = []
for outNode in aidge_model.get_output_nodes():
if DEVICE == "cuda":
outNode.get_operator().get_output(0).set_backend('cpu')
output_aidge = np.array(outNode.get_operator().get_output(0))
aidge_out_arr = output_aidge[0]
print("Aidge output: {}".format(output_aidge))
print("************* ONNX Inference ***************")
onnx_session = onnxruntime.InferenceSession(model_path)
# Check input shape compatibility
input_name = onnx_session.get_inputs()[0].name
if input.shape != tuple(onnx_session.get_inputs()[0].shape):
raise ValueError(f"Input shape mismatch. Expected: {onnx_session.get_inputs()[0].shape}, Got: {input.shape}")
# Run inference
output_name = onnx_session.get_outputs()[0].name
result = onnx_session.run([output_name], {input_name: input})
onnx_out_arr = result[0][0]
# Print the inference result
print("ONNX output: {}".format(result[0][0][:10]))
diff_arr = aidge_out_arr - onnx_out_arr
print("diff max ", np.max(diff_arr))
print("diff mean ", np.sum(diff_arr)/len(diff_arr))
percentage_array = (diff_arr / onnx_out_arr) * 100
print("diff % max ", np.max(percentage_array))
print("diff % mean ", np.sum(percentage_array)/len(percentage_array))
Edited by Houssem ROUIS