Skip to content
Snippets Groups Projects
Commit 28fe9a43 authored by Maxence Naud's avatar Maxence Naud
Browse files

Merge branch 'dev' into 'main'

Update Aidge export to take a graph view has an argument instead of a...

Closes aidge#112

See merge request eclipse/aidge/aidge_core!152
parents e3b48e55 4dcc63d2
No related branches found
No related tags found
No related merge requests found
Showing
with 567 additions and 1 deletion
include MANIFEST.in
include LICENSE
include README.md
recursive-include aidge_core *
include setup.py
include version.txt
......@@ -8,5 +8,6 @@ http://www.eclipse.org/legal/epl-2.0.
SPDX-License-Identifier: EPL-2.0
"""
from aidge_core.aidge_core import * # import so generated by PyBind
from aidge_core.export import ExportNode, generate_file, generate_str
from aidge_core.export_utils import ExportNode, generate_file, generate_str
import aidge_core.utils
from aidge_core.aidge_export_aidge import *
from pathlib import Path
# Constants
FILE = Path(__file__).resolve()
ROOT_EXPORT = FILE.parents[0]
from .operator_export import *
from .export import export
import aidge_core
import shutil
import os
from pathlib import Path
from .utils import supported_operators, OPERATORS_REGISTRY
from . import ROOT_EXPORT
from aidge_core import ExportNode, generate_file
def export(export_folder: str,
graph_view: aidge_core.GraphView,
enable_python_binding: bool = True,
):
export_folder_path = Path(export_folder)
export_name = export_folder_path.name
### Creating export folder ###
# Create export directory
os.makedirs(export_folder, exist_ok=True)
### Cpy static files ###
shutil.copytree(ROOT_EXPORT / "static/include",
export_folder_path / "include", dirs_exist_ok=True)
shutil.copytree(ROOT_EXPORT / "static/cmake",
export_folder_path / "cmake", dirs_exist_ok=True)
shutil.copyfile(ROOT_EXPORT / "static/CMakeLists.txt",
export_folder_path / "CMakeLists.txt")
shutil.copyfile(ROOT_EXPORT / "static/version.txt",
export_folder_path / "version.txt")
shutil.copyfile(ROOT_EXPORT / "static/README.md",
export_folder_path / "README.md")
shutil.copyfile(ROOT_EXPORT / "static/main.cpp",
export_folder_path / "main.cpp")
shutil.copyfile(ROOT_EXPORT / "static/export-config.cmake.in",
export_folder_path / f"{export_name}-config.cmake.in")
# Create project_name file
with open(export_folder_path / "project_name.txt", "w") as f:
f.write(export_name)
# Add files related to python binding if
if enable_python_binding:
os.makedirs(export_folder_path / "python_binding", exist_ok=True)
generate_file(
export_folder_path / "python_binding/pybind.cpp",
ROOT_EXPORT / "templates/pybind.jinja",
name=export_name,
)
# TODO: Add a main.py file ?
### Generating an export for each nodes and dnn file ###
list_configs = [] # List of headers to include in dnn.cpp to access attribute and parameters
list_actions = [] # List of string to construct graph
set_operator = set()
# Queue of Aidge nodes to explore, guarantee a topological exploration of the graph
open_nodes = list(graph_view.get_input_nodes())
# List of Aidge nodes already explored
closed_nodes = []
while open_nodes:
node = open_nodes.pop(0)
if node in closed_nodes:
continue # Node already converted, moving on ...
parents_not_converted = False
# Check all parents have been converted
for parent in node.get_parents():
if parent is not None and \
parent not in closed_nodes:
# If parents have not been converted, push back current node
if not parents_not_converted:
open_nodes.insert(0, node)
parents_not_converted = True
# Add to the stack the not converted parent as next node to convert
open_nodes.insert(0, parent)
if parents_not_converted:
continue
# Next nodes to treat are children of current node
open_nodes += list(node.get_children())
if node.type() in supported_operators():
set_operator.add(node.type())
op = OPERATORS_REGISTRY[node.type()](node)
# TODO: list_configs and list_actions don't need to be passed by argument
# Export the configuration
list_configs = op.export(export_folder_path, list_configs)
# Add forward kernel
list_actions = op.forward(list_actions)
else:
raise RuntimeError(f"Operator: {node.type()} is not supported")
closed_nodes.append(node)
# Generate full dnn.cpp
aidge_core.generate_file(
export_folder_path / "src/dnn.cpp",
ROOT_EXPORT / "templates/dnn.jinja",
headers=list_configs,
operators=set_operator,
actions=list_actions,
)
"""
Copyright (c) 2023 CEA-List
This program and the accompanying materials are made available under the
terms of the Eclipse Public License 2.0 which is available at
http://www.eclipse.org/legal/epl-2.0.
SPDX-License-Identifier: EPL-2.0
"""
from pathlib import Path
DIR_PATH = Path(__file__).parent
modules = [Path(module).stem for module in DIR_PATH.glob("*.py")]
__all__ = [ f for f in modules if f != "__init__"]
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import ExportNode, generate_file, generate_str
from pathlib import Path
@operator_register("Conv")
class Conv(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
include_path = f"attributes/{self.name}.hpp"
filepath = export_folder / f"include/{include_path}"
generate_file(
filepath,
ROOT_EXPORT / "templates/attributes/conv.jinja",
name=self.name,
**self.attributes
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT /"templates/graph_ctor/conv.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register,parse_node_input
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import ExportNode, generate_file, generate_str
from pathlib import Path
@operator_register("FC")
class FC(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
include_path = f"attributes/{self.name}.hpp"
filepath = export_folder / f"include/{include_path}"
generate_file(
filepath,
ROOT_EXPORT / "templates/attributes/fc.jinja",
name=self.name,
InChannels=self.inputs_dims[1][1],
OutChannels=self.operator.out_channels(),
**self.attributes
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/fc.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import ExportNode, generate_file, generate_str
from pathlib import Path
@operator_register("MaxPooling")
class MaxPooling(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
include_path = f"attributes/{self.name}.hpp"
filepath = export_folder / f"include/{include_path}"
generate_file(
filepath,
ROOT_EXPORT / "templates/attributes/maxpooling.jinja",
name=self.name,
**self.attributes
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/maxpooling.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import dtype, ExportNode, generate_file, generate_str
import numpy as np
from pathlib import Path
# Convert aidge datatype to C++ type
datatype_converter = {
dtype.float64 : "double",
dtype.float32 : "float",
dtype.float16 : "half_float::half",
dtype.int8 : "int8_t",
dtype.int16 : "int16_t",
dtype.int32 : "int32_t",
dtype.int64 : "int64_t",
dtype.uint8 : "uint8_t",
dtype.uint16 : "uint16_t",
dtype.uint32 : "uint32_t",
dtype.uint64 : "uint64_t"
}
@operator_register("Producer")
class Producer(ExportNode):
"""
If there is a standardization of the export operators
then this class should be just a inheritance of ProducerCPP
"""
def __init__(self, node):
super().__init__(node)
child, in_idx = self.node.output(0)[0]
self.tensor_name = f"{child.name()}_{in_idx}"
self.values = np.array(self.operator.get_output(0))
def export(self, export_folder:Path, list_configs:list):
assert(len(self.node.output(0)) == 1)
include_path = f"parameters/{self.tensor_name}.hpp"
filepath = export_folder / f"include/{include_path}"
aidge_tensor = self.operator.get_output(0)
aidge_type = aidge_tensor.dtype()
if aidge_type in datatype_converter:
datatype = datatype_converter[aidge_type]
else:
raise RuntimeError(f"No conversion found for data type {aidge_type}.")
generate_file(
filepath,
ROOT_EXPORT / "templates/parameter.jinja",
dims = aidge_tensor.dims(),
data_t = datatype, # TODO : get data from producer
name = self.tensor_name,
values = str(aidge_tensor)
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/producer.jinja",
name=self.name,
tensor_name=self.tensor_name,
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core import ExportNode, generate_str
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from pathlib import Path
@operator_register("ReLU")
class ReLU(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/relu.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core import ExportNode, generate_str
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from pathlib import Path
@operator_register("Sub")
class Sub(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/sub.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
cmake_minimum_required(VERSION 3.15)
file(STRINGS "${CMAKE_SOURCE_DIR}/version.txt" version)
file(STRINGS "${CMAKE_SOURCE_DIR}/project_name.txt" project)
message(STATUS "Project name: ${project}")
message(STATUS "Project version: ${version}")
# Note : project name is {project} and python module name is also {project}
set(module_name _${project}) # target name
project(${project})
set(CXX_STANDARD 14)
##############################################
# Define options
option(PYBIND "python binding" ON)
option(WERROR "Warning as error" OFF)
option(TEST "Enable tests" ON)
option(COVERAGE "Enable coverage" OFF)
option(ENABLE_ASAN "Enable ASan (AddressSanitizer) for runtime analysis of memory use (over/underflow, memory leak, ...)" OFF)
##############################################
# Import utils CMakeLists
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
include(PybindModuleCreation)
if(CMAKE_COMPILER_IS_GNUCXX AND COVERAGE)
Include(CodeCoverage)
endif()
##############################################
# Find system dependencies
find_package(aidge_core REQUIRED)
# Example of adding dependency to backend CPU
# find_package(aidge_backend_cpu REQUIRED)
##############################################
# Create target and set properties
file(GLOB_RECURSE src_files "src/*.cpp")
file(GLOB_RECURSE inc_files "include/*.hpp")
add_library(${module_name} ${src_files} ${inc_files})
target_link_libraries(${module_name}
PUBLIC
_aidge_core # _ is added because we link the target not the project
# _aidge_backend_cpu # Example of adding dependency to backend CPUs
)
#Set target properties
set_property(TARGET ${module_name} PROPERTY POSITION_INDEPENDENT_CODE ON)
if( ${ENABLE_ASAN} )
message("Building ${module_name} with ASAN.")
set(SANITIZE_FLAGS -fsanitize=address -fno-omit-frame-pointer)
target_link_libraries(${module_name}
PUBLIC
-fsanitize=address
)
target_compile_options(${module_name}
PRIVATE
${SANITIZE_FLAGS}
)
endif()
target_include_directories(${module_name}
PUBLIC
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
)
# PYTHON BINDING
if (PYBIND)
generate_python_binding(${project} ${module_name})
# Handles Python + pybind11 headers dependencies
target_link_libraries(${module_name}
PUBLIC
pybind11::pybind11
)
endif()
target_compile_features(${module_name} PRIVATE cxx_std_14)
target_compile_options(${module_name} PRIVATE
$<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
-Wall -Wextra -Wold-style-cast -Winline -pedantic -Werror=narrowing -Wshadow $<$<BOOL:${WERROR}>:-Werror>>)
target_compile_options(${module_name} PRIVATE
$<$<CXX_COMPILER_ID:MSVC>:
/W4>)
if(CMAKE_COMPILER_IS_GNUCXX AND COVERAGE)
append_coverage_compiler_flags()
endif()
##############################################
# Installation instructions
include(GNUInstallDirs)
set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/${project})
install(TARGETS ${module_name} EXPORT ${project}-targets
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)
install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
#Export the targets to a script
install(EXPORT ${project}-targets
FILE "${project}-targets.cmake"
DESTINATION ${INSTALL_CONFIGDIR}
COMPONENT ${module_name}
)
#Create a ConfigVersion.cmake file
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config-version.cmake"
VERSION ${version}
COMPATIBILITY AnyNewerVersion
)
configure_package_config_file("${project}-config.cmake.in"
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config.cmake"
INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
)
#Install the config, configversion and custom find modules
install(FILES
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config-version.cmake"
DESTINATION ${INSTALL_CONFIGDIR}
)
##############################################
## Exporting from the build tree
export(EXPORT ${project}-targets
FILE "${CMAKE_CURRENT_BINARY_DIR}/${project}-targets.cmake")
# Compile executable
add_executable(main main.cpp)
target_link_libraries(main PUBLIC _aidge_core ${module_name})
To compile:
> mkdir build && cd build
> cmake -DCMAKE_INSTALL_PREFIX:PATH=/data1/is156025/cm264821/anaconda3/envs/aidge_demo/lib/libAidge ..
> make all install
function(generate_python_binding name target_to_bind)
add_definitions(-DPYBIND)
Include(FetchContent)
FetchContent_Declare(
PyBind11
GIT_REPOSITORY https://github.com/pybind/pybind11.git
GIT_TAG v2.10.4 # or a later release
)
# Use the New FindPython mode, recommanded. Requires CMake 3.15+
find_package(Python COMPONENTS Interpreter Development)
FetchContent_MakeAvailable(PyBind11)
message(STATUS "Creating binding for module ${name}")
file(GLOB_RECURSE pybind_src_files "python_binding/*.cpp")
pybind11_add_module(${name} MODULE ${pybind_src_files} "NO_EXTRAS") # NO EXTRA recquired for pip install
target_include_directories(${name} PUBLIC "python_binding")
target_link_libraries(${name} PUBLIC ${target_to_bind})
endfunction()
include(${CMAKE_CURRENT_LIST_DIR}/aidge_backend_cpu-config-version.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/aidge_backend_cpu-targets.cmake)
#ifndef DNN_HPP
#define DNN_HPP
#include <aidge/graph/GraphView.hpp>
/**
* @brief This file contains all of what is related to the construction of the
* neural network
*
*/
/**
* @brief This function generate the exported Aidge::GraphView.
*
* @return std::shared_ptr<Aidge::GraphView>
*/
std::shared_ptr<Aidge::GraphView> generateModel();
#endif /* DNN_HPP */
#include <iostream>
#include <aidge/backend/cpu.hpp>
#include "include/dnn.hpp"
int main()
{
std::cout << "BEGIN" << std::endl;
std::shared_ptr<Aidge::GraphView> graph = generateModel();
std::cout << "END" << std::endl;
return 0;
}
export
#include <pybind11/pybind11.h>
#include "dnn.hpp"
namespace py = pybind11;
void init_export(py::module& m){
m.def("generate_model", generateModel);
}
PYBIND11_MODULE(export, m) {
init_export(m);
}
0.0.0
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment