Merge branch 'dev' into nobias

MANIFEST.in

+include MANIFEST.in
+include LICENSE
+include README.md
+recursive-include aidge_core *
+include setup.py
+include version.txt

aidge_core/__init__.py

@@ -8,5 +8,6 @@ http://www.eclipse.org/legal/epl-2.0.
 SPDX-License-Identifier: EPL-2.0
 """
 from aidge_core.aidge_core import * # import so generated by PyBind
-from aidge_core.export import ExportNode, generate_file, generate_str
+from aidge_core.export_utils import ExportNode, generate_file, generate_str
 import aidge_core.utils
+from aidge_core.aidge_export_aidge import *

aidge_core/aidge_export_aidge/__init__.py

+from pathlib import Path
+
+# Constants
+FILE = Path(__file__).resolve()
+ROOT_EXPORT = FILE.parents[0]
+
+from .operator_export import *
+from .export import export

aidge_core/aidge_export_aidge/export.py

+import aidge_core
+import shutil
+import os
+from pathlib import Path
+from .utils import supported_operators, OPERATORS_REGISTRY
+from . import ROOT_EXPORT
+
+
+from aidge_core import ExportNode, generate_file
+
+
+
+def export(export_folder: str,
+           graph_view: aidge_core.GraphView,
+           enable_python_binding: bool = True,
+           ):
+    export_folder_path = Path(export_folder)
+    export_name = export_folder_path.name
+
+    ### Creating export folder ###
+    # Create export directory
+    os.makedirs(export_folder, exist_ok=True)
+
+    ### Cpy static files ###
+    shutil.copytree(ROOT_EXPORT / "static/include",
+                    export_folder_path / "include", dirs_exist_ok=True)
+    shutil.copytree(ROOT_EXPORT / "static/cmake",
+                    export_folder_path / "cmake", dirs_exist_ok=True)
+    shutil.copyfile(ROOT_EXPORT / "static/CMakeLists.txt",
+                    export_folder_path / "CMakeLists.txt")
+    shutil.copyfile(ROOT_EXPORT / "static/version.txt",
+                    export_folder_path / "version.txt")
+    shutil.copyfile(ROOT_EXPORT / "static/README.md",
+                    export_folder_path / "README.md")
+    shutil.copyfile(ROOT_EXPORT / "static/main.cpp",
+                    export_folder_path / "main.cpp")
+    shutil.copyfile(ROOT_EXPORT / "static/export-config.cmake.in",
+                    export_folder_path / f"{export_name}-config.cmake.in")
+
+    # Create project_name file
+    with open(export_folder_path / "project_name.txt", "w") as f:
+        f.write(export_name)
+
+    # Add files related to python binding if
+    if enable_python_binding:
+        os.makedirs(export_folder_path / "python_binding", exist_ok=True)
+        generate_file(
+            export_folder_path / "python_binding/pybind.cpp",
+            ROOT_EXPORT / "templates/pybind.jinja",
+            name=export_name,
+        )
+        # TODO: Add a main.py file ?
+
+    ### Generating an export for each nodes and dnn file ###
+    list_configs = []  # List of headers to include in dnn.cpp to access attribute and parameters
+    list_actions = []  # List of string to construct graph
+    set_operator = set()
+    # Queue of Aidge nodes to explore, guarantee a topological exploration of the graph
+    open_nodes = list(graph_view.get_input_nodes())
+    # List of Aidge nodes already explored
+    closed_nodes = []
+
+    while open_nodes:
+        node = open_nodes.pop(0)
+        if node in closed_nodes:
+            continue  # Node already converted, moving on ...
+        parents_not_converted = False
+        # Check all parents have been converted
+        for parent in node.get_parents():
+            if parent is not None and \
+                    parent not in closed_nodes:
+                # If parents have not been converted, push back current node
+                if not parents_not_converted:
+                    open_nodes.insert(0, node)
+                    parents_not_converted = True
+                # Add to the stack the not converted parent as next node to convert
+                open_nodes.insert(0, parent)
+        if parents_not_converted:
+
+            continue
+        # Next nodes to treat are children of current node
+        open_nodes += list(node.get_children())
+
+        if node.type() in supported_operators():
+            set_operator.add(node.type())
+            op = OPERATORS_REGISTRY[node.type()](node)
+
+            # TODO: list_configs and list_actions don't need to be passed by argument
+            # Export the configuration
+            list_configs = op.export(export_folder_path, list_configs)
+
+            # Add forward kernel
+            list_actions = op.forward(list_actions)
+        else:
+            raise RuntimeError(f"Operator: {node.type()} is not supported")
+        closed_nodes.append(node)
+    # Generate full dnn.cpp
+    aidge_core.generate_file(
+        export_folder_path / "src/dnn.cpp",
+        ROOT_EXPORT / "templates/dnn.jinja",
+        headers=list_configs,
+        operators=set_operator,
+        actions=list_actions,
+    )

aidge_core/aidge_export_aidge/operator_export/__init__.py

+"""
+Copyright (c) 2023 CEA-List
+
+This program and the accompanying materials are made available under the
+terms of the Eclipse Public License 2.0 which is available at
+http://www.eclipse.org/legal/epl-2.0.
+
+SPDX-License-Identifier: EPL-2.0
+"""
+from pathlib import Path
+
+DIR_PATH = Path(__file__).parent
+modules = [Path(module).stem for module in DIR_PATH.glob("*.py")]
+__all__ = [ f for f in modules if f != "__init__"]

aidge_core/aidge_export_aidge/operator_export/conv.py

+from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
+from aidge_core.aidge_export_aidge import ROOT_EXPORT
+from aidge_core import ExportNode, generate_file, generate_str
+from pathlib import Path
+
+@operator_register("Conv")
+class Conv(ExportNode):
+    def __init__(self, node):
+        super().__init__(node)
+
+    def export(self, export_folder:Path, list_configs:list):
+        include_path = f"attributes/{self.name}.hpp"
+        filepath = export_folder / f"include/{include_path}"
+
+        generate_file(
+            filepath,
+            ROOT_EXPORT / "templates/attributes/conv.jinja",
+            name=self.name,
+            **self.attributes
+        )
+        list_configs.append(include_path)
+        return list_configs
+
+    def forward(self, list_actions:list):
+        list_actions.append(generate_str(
+            ROOT_EXPORT /"templates/graph_ctor/conv.jinja",
+            name=self.name,
+            inputs=parse_node_input(self.node.inputs()),
+            **self.attributes
+        ))
+        return list_actions

aidge_core/aidge_export_aidge/operator_export/fc.py

+from aidge_core.aidge_export_aidge.utils import operator_register,parse_node_input
+from aidge_core.aidge_export_aidge import ROOT_EXPORT
+from aidge_core import ExportNode, generate_file, generate_str
+from pathlib import Path
+
+@operator_register("FC")
+class FC(ExportNode):
+    def __init__(self, node):
+        super().__init__(node)
+
+
+    def export(self, export_folder:Path, list_configs:list):
+
+
+        include_path = f"attributes/{self.name}.hpp"
+        filepath = export_folder / f"include/{include_path}"
+
+
+        generate_file(
+            filepath,
+            ROOT_EXPORT / "templates/attributes/fc.jinja",
+            name=self.name,
+            InChannels=self.inputs_dims[1][1],
+            OutChannels=self.operator.out_channels(),
+            **self.attributes
+        )
+        list_configs.append(include_path)
+        return list_configs
+
+    def forward(self, list_actions:list):
+        list_actions.append(generate_str(
+            ROOT_EXPORT / "templates/graph_ctor/fc.jinja",
+            name=self.name,
+            inputs=parse_node_input(self.node.inputs()),
+            **self.attributes
+        ))
+        return list_actions

aidge_core/aidge_export_aidge/operator_export/maxpooling.py

+from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
+from aidge_core.aidge_export_aidge import ROOT_EXPORT
+from aidge_core import ExportNode, generate_file, generate_str
+from pathlib import Path
+
+@operator_register("MaxPooling")
+class MaxPooling(ExportNode):
+    def __init__(self, node):
+        super().__init__(node)
+
+
+    def export(self, export_folder:Path, list_configs:list):
+        include_path = f"attributes/{self.name}.hpp"
+        filepath = export_folder / f"include/{include_path}"
+
+        generate_file(
+            filepath,
+            ROOT_EXPORT / "templates/attributes/maxpooling.jinja",
+            name=self.name,
+            **self.attributes
+        )
+        list_configs.append(include_path)
+        return list_configs
+
+    def forward(self, list_actions:list):
+        list_actions.append(generate_str(
+            ROOT_EXPORT / "templates/graph_ctor/maxpooling.jinja",
+            name=self.name,
+            inputs=parse_node_input(self.node.inputs()),
+            **self.attributes
+        ))
+        return list_actions

aidge_core/aidge_export_aidge/operator_export/producer.py

+from aidge_core.aidge_export_aidge.utils import operator_register
+from aidge_core.aidge_export_aidge import ROOT_EXPORT
+from aidge_core import DataType, ExportNode, generate_file, generate_str
+import numpy as np
+from pathlib import Path
+
+# Convert aidge datatype to C++ type
+datatype_converter = {
+    DataType.Float64 : "double",
+    DataType.Float32 : "float",
+    DataType.Float16 : "half_float::half",
+    DataType.Int8    : "int8_t",
+    DataType.Int16   : "int16_t",
+    DataType.Int32   : "int32_t",
+    DataType.Int64   : "int64_t",
+    DataType.UInt8   : "uint8_t",
+    DataType.UInt16  : "uint16_t",
+    DataType.UInt32  : "uint32_t",
+    DataType.UInt64  : "uint64_t"
+}
+
+
+@operator_register("Producer")
+class Producer(ExportNode):
+    """
+    If there is a standardization of the export operators
+    then this class should be just a inheritance of ProducerCPP
+    """
+    def __init__(self, node):
+        super().__init__(node)
+        child, in_idx = self.node.output(0)[0]
+        self.tensor_name = f"{child.name()}_{in_idx}"
+        self.values = np.array(self.operator.get_output(0))
+
+    def export(self, export_folder:Path, list_configs:list):
+        assert(len(self.node.output(0)) == 1)
+
+        include_path = f"parameters/{self.tensor_name}.hpp"
+        filepath = export_folder / f"include/{include_path}"
+
+        aidge_tensor = self.operator.get_output(0)
+        aidge_type = aidge_tensor.dtype()
+        if aidge_type in datatype_converter:
+            datatype = datatype_converter[aidge_type]
+        else:
+            raise RuntimeError(f"No conversion found for data type {aidge_type}.")
+        generate_file(
+            filepath,
+            ROOT_EXPORT / "templates/parameter.jinja",
+            dims = aidge_tensor.dims(),
+            data_t = datatype, # TODO : get data from producer
+            name = self.tensor_name,
+            values = str(aidge_tensor)
+        )
+        list_configs.append(include_path)
+        return list_configs
+
+    def forward(self, list_actions:list):
+        list_actions.append(generate_str(
+            ROOT_EXPORT / "templates/graph_ctor/producer.jinja",
+            name=self.name,
+            tensor_name=self.tensor_name,
+            **self.attributes
+        ))
+        return list_actions

aidge_core/aidge_export_aidge/operator_export/relu.py

+from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
+from aidge_core import ExportNode, generate_str
+from aidge_core.aidge_export_aidge import ROOT_EXPORT
+from pathlib import Path
+
+@operator_register("ReLU")
+class ReLU(ExportNode):
+    def __init__(self, node):
+        super().__init__(node)
+
+    def export(self, export_folder:Path, list_configs:list):
+        return list_configs
+
+    def forward(self, list_actions:list):
+        list_actions.append(generate_str(
+            ROOT_EXPORT / "templates/graph_ctor/relu.jinja",
+            name=self.name,
+            inputs=parse_node_input(self.node.inputs()),
+            **self.attributes
+        ))
+        return list_actions

aidge_core/aidge_export_aidge/operator_export/sub.py

+from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
+from aidge_core import ExportNode, generate_str
+from aidge_core.aidge_export_aidge import ROOT_EXPORT
+from pathlib import Path
+
+@operator_register("Sub")
+class Sub(ExportNode):
+    def __init__(self, node):
+        super().__init__(node)
+
+    def export(self, export_folder:Path, list_configs:list):
+        return list_configs
+
+    def forward(self, list_actions:list):
+        list_actions.append(generate_str(
+            ROOT_EXPORT / "templates/graph_ctor/sub.jinja",
+            name=self.name,
+            inputs=parse_node_input(self.node.inputs()),
+            **self.attributes
+        ))
+        return list_actions

aidge_core/aidge_export_aidge/static/CMakeLists.txt

+cmake_minimum_required(VERSION 3.15)
+
+
+file(STRINGS "${CMAKE_SOURCE_DIR}/version.txt" version)
+file(STRINGS "${CMAKE_SOURCE_DIR}/project_name.txt" project)
+
+message(STATUS "Project name: ${project}")
+message(STATUS "Project version: ${version}")
+
+# Note : project name is {project} and python module name is also {project}
+set(module_name _${project}) # target name
+
+project(${project})
+set(CXX_STANDARD 14)
+
+##############################################
+# Define options
+option(PYBIND "python binding" ON)
+option(WERROR "Warning as error" OFF)
+option(TEST "Enable tests" ON)
+option(COVERAGE "Enable coverage" OFF)
+option(ENABLE_ASAN "Enable ASan (AddressSanitizer) for runtime analysis of memory use (over/underflow, memory leak, ...)" OFF)
+
+##############################################
+# Import utils CMakeLists
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
+include(PybindModuleCreation)
+
+if(CMAKE_COMPILER_IS_GNUCXX AND COVERAGE)
+    Include(CodeCoverage)
+endif()
+
+##############################################
+# Find system dependencies
+find_package(aidge_core REQUIRED)
+# Example of adding dependency to backend CPU
+# find_package(aidge_backend_cpu REQUIRED)
+
+##############################################
+# Create target and set properties
+file(GLOB_RECURSE src_files "src/*.cpp")
+file(GLOB_RECURSE inc_files "include/*.hpp")
+
+add_library(${module_name} ${src_files} ${inc_files})
+target_link_libraries(${module_name}
+    PUBLIC
+        _aidge_core # _ is added because we link the target not the project
+        # _aidge_backend_cpu  # Example of adding dependency to backend CPUs
+)
+
+#Set target properties
+set_property(TARGET ${module_name} PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+if( ${ENABLE_ASAN} )
+    message("Building ${module_name} with ASAN.")
+    set(SANITIZE_FLAGS -fsanitize=address -fno-omit-frame-pointer)
+    target_link_libraries(${module_name}
+        PUBLIC
+            -fsanitize=address
+    )
+    target_compile_options(${module_name}
+        PRIVATE
+            ${SANITIZE_FLAGS}
+    )
+endif()
+
+target_include_directories(${module_name}
+    PUBLIC
+        $<INSTALL_INTERFACE:include>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+# PYTHON BINDING
+if (PYBIND)
+    generate_python_binding(${project} ${module_name})
+
+    # Handles Python + pybind11 headers dependencies
+    target_link_libraries(${module_name}
+        PUBLIC
+            pybind11::pybind11
+        )
+endif()
+
+target_compile_features(${module_name} PRIVATE cxx_std_14)
+
+target_compile_options(${module_name} PRIVATE
+    $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+    -Wall -Wextra -Wold-style-cast -Winline -pedantic -Werror=narrowing -Wshadow $<$<BOOL:${WERROR}>:-Werror>>)
+target_compile_options(${module_name} PRIVATE
+    $<$<CXX_COMPILER_ID:MSVC>:
+    /W4>)
+
+if(CMAKE_COMPILER_IS_GNUCXX AND COVERAGE)
+    append_coverage_compiler_flags()
+endif()
+
+##############################################
+# Installation instructions
+
+include(GNUInstallDirs)
+set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/${project})
+
+install(TARGETS ${module_name} EXPORT ${project}-targets
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+)
+
+install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+#Export the targets to a script
+
+install(EXPORT ${project}-targets
+ FILE "${project}-targets.cmake"
+ DESTINATION ${INSTALL_CONFIGDIR}
+ COMPONENT ${module_name}
+)
+
+#Create a ConfigVersion.cmake file
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${project}-config-version.cmake"
+    VERSION ${version}
+    COMPATIBILITY AnyNewerVersion
+)
+
+configure_package_config_file("${project}-config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${project}-config.cmake"
+    INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
+)
+
+#Install the config, configversion and custom find modules
+install(FILES
+    "${CMAKE_CURRENT_BINARY_DIR}/${project}-config.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${project}-config-version.cmake"
+    DESTINATION ${INSTALL_CONFIGDIR}
+)
+
+##############################################
+## Exporting from the build tree
+export(EXPORT ${project}-targets
+    FILE "${CMAKE_CURRENT_BINARY_DIR}/${project}-targets.cmake")
+
+# Compile executable
+add_executable(main main.cpp)
+target_link_libraries(main PUBLIC _aidge_core ${module_name})

aidge_core/aidge_export_aidge/static/README.md

+To compile:
+
+> mkdir build && cd build
+> cmake -DCMAKE_INSTALL_PREFIX:PATH=/data1/is156025/cm264821/anaconda3/envs/aidge_demo/lib/libAidge ..
+> make all install

aidge_core/aidge_export_aidge/static/cmake/PybindModuleCreation.cmake

+function(generate_python_binding name target_to_bind) 
+    add_definitions(-DPYBIND)
+    Include(FetchContent)
+
+    FetchContent_Declare(
+    PyBind11
+    GIT_REPOSITORY https://github.com/pybind/pybind11.git
+    GIT_TAG        v2.10.4 # or a later release
+    )
+
+    # Use the New FindPython mode, recommanded. Requires CMake 3.15+
+    find_package(Python COMPONENTS Interpreter Development)
+    FetchContent_MakeAvailable(PyBind11)
+
+    message(STATUS "Creating binding for module ${name}")
+    file(GLOB_RECURSE pybind_src_files "python_binding/*.cpp")
+
+    pybind11_add_module(${name} MODULE ${pybind_src_files} "NO_EXTRAS") # NO EXTRA recquired for pip install
+    target_include_directories(${name} PUBLIC "python_binding")
+    target_link_libraries(${name} PUBLIC ${target_to_bind})
+endfunction()

aidge_core/aidge_export_aidge/static/export-config.cmake.in

+include(${CMAKE_CURRENT_LIST_DIR}/aidge_backend_cpu-config-version.cmake)
+
+include(${CMAKE_CURRENT_LIST_DIR}/aidge_backend_cpu-targets.cmake)

aidge_core/aidge_export_aidge/static/include/dnn.hpp

+#ifndef DNN_HPP
+#define DNN_HPP
+#include <aidge/graph/GraphView.hpp>
+/**
+ * @brief This file contains all of what is related to the construction of the
+ * neural network
+ *
+ */
+
+/**
+ * @brief This function generate the exported Aidge::GraphView.
+ *
+ * @return std::shared_ptr<Aidge::GraphView>
+ */
+std::shared_ptr<Aidge::GraphView> generateModel();
+
+#endif /* DNN_HPP */

aidge_core/aidge_export_aidge/static/main.cpp

+#include <iostream>
+#include <aidge/backend/cpu.hpp>
+
+#include "include/dnn.hpp"
+
+int main()
+{
+
+    std::cout << "BEGIN" << std::endl;
+
+    std::shared_ptr<Aidge::GraphView> graph = generateModel();
+
+    std::cout << "END" << std::endl;
+
+    return 0;
+}

aidge_core/aidge_export_aidge/static/project_name.txt

+export

aidge_core/aidge_export_aidge/static/python_binding/pybind.cpp

+#include <pybind11/pybind11.h>
+
+#include "dnn.hpp"
+
+namespace py = pybind11;
+
+
+void init_export(py::module& m){
+    m.def("generate_model", generateModel);
+}
+
+PYBIND11_MODULE(export, m) {
+    init_export(m);
+}

aidge_core/aidge_export_aidge/static/version.txt

+0.0.0