Skip to content
Snippets Groups Projects
Commit 294e6294 authored by Olivier BICHLER's avatar Olivier BICHLER
Browse files

Merge branch 'dev' into nobias

parents 17d45782 a3afd376
No related branches found
No related tags found
2 merge requests!152Update Aidge export to take a graph view has an argument instead of a...,!143Multiple refactors
Showing
with 567 additions and 1 deletion
include MANIFEST.in
include LICENSE
include README.md
recursive-include aidge_core *
include setup.py
include version.txt
......@@ -8,5 +8,6 @@ http://www.eclipse.org/legal/epl-2.0.
SPDX-License-Identifier: EPL-2.0
"""
from aidge_core.aidge_core import * # import so generated by PyBind
from aidge_core.export import ExportNode, generate_file, generate_str
from aidge_core.export_utils import ExportNode, generate_file, generate_str
import aidge_core.utils
from aidge_core.aidge_export_aidge import *
from pathlib import Path
# Constants
FILE = Path(__file__).resolve()
ROOT_EXPORT = FILE.parents[0]
from .operator_export import *
from .export import export
import aidge_core
import shutil
import os
from pathlib import Path
from .utils import supported_operators, OPERATORS_REGISTRY
from . import ROOT_EXPORT
from aidge_core import ExportNode, generate_file
def export(export_folder: str,
graph_view: aidge_core.GraphView,
enable_python_binding: bool = True,
):
export_folder_path = Path(export_folder)
export_name = export_folder_path.name
### Creating export folder ###
# Create export directory
os.makedirs(export_folder, exist_ok=True)
### Cpy static files ###
shutil.copytree(ROOT_EXPORT / "static/include",
export_folder_path / "include", dirs_exist_ok=True)
shutil.copytree(ROOT_EXPORT / "static/cmake",
export_folder_path / "cmake", dirs_exist_ok=True)
shutil.copyfile(ROOT_EXPORT / "static/CMakeLists.txt",
export_folder_path / "CMakeLists.txt")
shutil.copyfile(ROOT_EXPORT / "static/version.txt",
export_folder_path / "version.txt")
shutil.copyfile(ROOT_EXPORT / "static/README.md",
export_folder_path / "README.md")
shutil.copyfile(ROOT_EXPORT / "static/main.cpp",
export_folder_path / "main.cpp")
shutil.copyfile(ROOT_EXPORT / "static/export-config.cmake.in",
export_folder_path / f"{export_name}-config.cmake.in")
# Create project_name file
with open(export_folder_path / "project_name.txt", "w") as f:
f.write(export_name)
# Add files related to python binding if
if enable_python_binding:
os.makedirs(export_folder_path / "python_binding", exist_ok=True)
generate_file(
export_folder_path / "python_binding/pybind.cpp",
ROOT_EXPORT / "templates/pybind.jinja",
name=export_name,
)
# TODO: Add a main.py file ?
### Generating an export for each nodes and dnn file ###
list_configs = [] # List of headers to include in dnn.cpp to access attribute and parameters
list_actions = [] # List of string to construct graph
set_operator = set()
# Queue of Aidge nodes to explore, guarantee a topological exploration of the graph
open_nodes = list(graph_view.get_input_nodes())
# List of Aidge nodes already explored
closed_nodes = []
while open_nodes:
node = open_nodes.pop(0)
if node in closed_nodes:
continue # Node already converted, moving on ...
parents_not_converted = False
# Check all parents have been converted
for parent in node.get_parents():
if parent is not None and \
parent not in closed_nodes:
# If parents have not been converted, push back current node
if not parents_not_converted:
open_nodes.insert(0, node)
parents_not_converted = True
# Add to the stack the not converted parent as next node to convert
open_nodes.insert(0, parent)
if parents_not_converted:
continue
# Next nodes to treat are children of current node
open_nodes += list(node.get_children())
if node.type() in supported_operators():
set_operator.add(node.type())
op = OPERATORS_REGISTRY[node.type()](node)
# TODO: list_configs and list_actions don't need to be passed by argument
# Export the configuration
list_configs = op.export(export_folder_path, list_configs)
# Add forward kernel
list_actions = op.forward(list_actions)
else:
raise RuntimeError(f"Operator: {node.type()} is not supported")
closed_nodes.append(node)
# Generate full dnn.cpp
aidge_core.generate_file(
export_folder_path / "src/dnn.cpp",
ROOT_EXPORT / "templates/dnn.jinja",
headers=list_configs,
operators=set_operator,
actions=list_actions,
)
"""
Copyright (c) 2023 CEA-List
This program and the accompanying materials are made available under the
terms of the Eclipse Public License 2.0 which is available at
http://www.eclipse.org/legal/epl-2.0.
SPDX-License-Identifier: EPL-2.0
"""
from pathlib import Path
DIR_PATH = Path(__file__).parent
modules = [Path(module).stem for module in DIR_PATH.glob("*.py")]
__all__ = [ f for f in modules if f != "__init__"]
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import ExportNode, generate_file, generate_str
from pathlib import Path
@operator_register("Conv")
class Conv(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
include_path = f"attributes/{self.name}.hpp"
filepath = export_folder / f"include/{include_path}"
generate_file(
filepath,
ROOT_EXPORT / "templates/attributes/conv.jinja",
name=self.name,
**self.attributes
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT /"templates/graph_ctor/conv.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register,parse_node_input
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import ExportNode, generate_file, generate_str
from pathlib import Path
@operator_register("FC")
class FC(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
include_path = f"attributes/{self.name}.hpp"
filepath = export_folder / f"include/{include_path}"
generate_file(
filepath,
ROOT_EXPORT / "templates/attributes/fc.jinja",
name=self.name,
InChannels=self.inputs_dims[1][1],
OutChannels=self.operator.out_channels(),
**self.attributes
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/fc.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import ExportNode, generate_file, generate_str
from pathlib import Path
@operator_register("MaxPooling")
class MaxPooling(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
include_path = f"attributes/{self.name}.hpp"
filepath = export_folder / f"include/{include_path}"
generate_file(
filepath,
ROOT_EXPORT / "templates/attributes/maxpooling.jinja",
name=self.name,
**self.attributes
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/maxpooling.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from aidge_core import DataType, ExportNode, generate_file, generate_str
import numpy as np
from pathlib import Path
# Convert aidge datatype to C++ type
datatype_converter = {
DataType.Float64 : "double",
DataType.Float32 : "float",
DataType.Float16 : "half_float::half",
DataType.Int8 : "int8_t",
DataType.Int16 : "int16_t",
DataType.Int32 : "int32_t",
DataType.Int64 : "int64_t",
DataType.UInt8 : "uint8_t",
DataType.UInt16 : "uint16_t",
DataType.UInt32 : "uint32_t",
DataType.UInt64 : "uint64_t"
}
@operator_register("Producer")
class Producer(ExportNode):
"""
If there is a standardization of the export operators
then this class should be just a inheritance of ProducerCPP
"""
def __init__(self, node):
super().__init__(node)
child, in_idx = self.node.output(0)[0]
self.tensor_name = f"{child.name()}_{in_idx}"
self.values = np.array(self.operator.get_output(0))
def export(self, export_folder:Path, list_configs:list):
assert(len(self.node.output(0)) == 1)
include_path = f"parameters/{self.tensor_name}.hpp"
filepath = export_folder / f"include/{include_path}"
aidge_tensor = self.operator.get_output(0)
aidge_type = aidge_tensor.dtype()
if aidge_type in datatype_converter:
datatype = datatype_converter[aidge_type]
else:
raise RuntimeError(f"No conversion found for data type {aidge_type}.")
generate_file(
filepath,
ROOT_EXPORT / "templates/parameter.jinja",
dims = aidge_tensor.dims(),
data_t = datatype, # TODO : get data from producer
name = self.tensor_name,
values = str(aidge_tensor)
)
list_configs.append(include_path)
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/producer.jinja",
name=self.name,
tensor_name=self.tensor_name,
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core import ExportNode, generate_str
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from pathlib import Path
@operator_register("ReLU")
class ReLU(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/relu.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
from aidge_core.aidge_export_aidge.utils import operator_register, parse_node_input
from aidge_core import ExportNode, generate_str
from aidge_core.aidge_export_aidge import ROOT_EXPORT
from pathlib import Path
@operator_register("Sub")
class Sub(ExportNode):
def __init__(self, node):
super().__init__(node)
def export(self, export_folder:Path, list_configs:list):
return list_configs
def forward(self, list_actions:list):
list_actions.append(generate_str(
ROOT_EXPORT / "templates/graph_ctor/sub.jinja",
name=self.name,
inputs=parse_node_input(self.node.inputs()),
**self.attributes
))
return list_actions
cmake_minimum_required(VERSION 3.15)
file(STRINGS "${CMAKE_SOURCE_DIR}/version.txt" version)
file(STRINGS "${CMAKE_SOURCE_DIR}/project_name.txt" project)
message(STATUS "Project name: ${project}")
message(STATUS "Project version: ${version}")
# Note : project name is {project} and python module name is also {project}
set(module_name _${project}) # target name
project(${project})
set(CXX_STANDARD 14)
##############################################
# Define options
option(PYBIND "python binding" ON)
option(WERROR "Warning as error" OFF)
option(TEST "Enable tests" ON)
option(COVERAGE "Enable coverage" OFF)
option(ENABLE_ASAN "Enable ASan (AddressSanitizer) for runtime analysis of memory use (over/underflow, memory leak, ...)" OFF)
##############################################
# Import utils CMakeLists
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
include(PybindModuleCreation)
if(CMAKE_COMPILER_IS_GNUCXX AND COVERAGE)
Include(CodeCoverage)
endif()
##############################################
# Find system dependencies
find_package(aidge_core REQUIRED)
# Example of adding dependency to backend CPU
# find_package(aidge_backend_cpu REQUIRED)
##############################################
# Create target and set properties
file(GLOB_RECURSE src_files "src/*.cpp")
file(GLOB_RECURSE inc_files "include/*.hpp")
add_library(${module_name} ${src_files} ${inc_files})
target_link_libraries(${module_name}
PUBLIC
_aidge_core # _ is added because we link the target not the project
# _aidge_backend_cpu # Example of adding dependency to backend CPUs
)
#Set target properties
set_property(TARGET ${module_name} PROPERTY POSITION_INDEPENDENT_CODE ON)
if( ${ENABLE_ASAN} )
message("Building ${module_name} with ASAN.")
set(SANITIZE_FLAGS -fsanitize=address -fno-omit-frame-pointer)
target_link_libraries(${module_name}
PUBLIC
-fsanitize=address
)
target_compile_options(${module_name}
PRIVATE
${SANITIZE_FLAGS}
)
endif()
target_include_directories(${module_name}
PUBLIC
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
)
# PYTHON BINDING
if (PYBIND)
generate_python_binding(${project} ${module_name})
# Handles Python + pybind11 headers dependencies
target_link_libraries(${module_name}
PUBLIC
pybind11::pybind11
)
endif()
target_compile_features(${module_name} PRIVATE cxx_std_14)
target_compile_options(${module_name} PRIVATE
$<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
-Wall -Wextra -Wold-style-cast -Winline -pedantic -Werror=narrowing -Wshadow $<$<BOOL:${WERROR}>:-Werror>>)
target_compile_options(${module_name} PRIVATE
$<$<CXX_COMPILER_ID:MSVC>:
/W4>)
if(CMAKE_COMPILER_IS_GNUCXX AND COVERAGE)
append_coverage_compiler_flags()
endif()
##############################################
# Installation instructions
include(GNUInstallDirs)
set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/${project})
install(TARGETS ${module_name} EXPORT ${project}-targets
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)
install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
#Export the targets to a script
install(EXPORT ${project}-targets
FILE "${project}-targets.cmake"
DESTINATION ${INSTALL_CONFIGDIR}
COMPONENT ${module_name}
)
#Create a ConfigVersion.cmake file
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config-version.cmake"
VERSION ${version}
COMPATIBILITY AnyNewerVersion
)
configure_package_config_file("${project}-config.cmake.in"
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config.cmake"
INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
)
#Install the config, configversion and custom find modules
install(FILES
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/${project}-config-version.cmake"
DESTINATION ${INSTALL_CONFIGDIR}
)
##############################################
## Exporting from the build tree
export(EXPORT ${project}-targets
FILE "${CMAKE_CURRENT_BINARY_DIR}/${project}-targets.cmake")
# Compile executable
add_executable(main main.cpp)
target_link_libraries(main PUBLIC _aidge_core ${module_name})
To compile:
> mkdir build && cd build
> cmake -DCMAKE_INSTALL_PREFIX:PATH=/data1/is156025/cm264821/anaconda3/envs/aidge_demo/lib/libAidge ..
> make all install
function(generate_python_binding name target_to_bind)
add_definitions(-DPYBIND)
Include(FetchContent)
FetchContent_Declare(
PyBind11
GIT_REPOSITORY https://github.com/pybind/pybind11.git
GIT_TAG v2.10.4 # or a later release
)
# Use the New FindPython mode, recommanded. Requires CMake 3.15+
find_package(Python COMPONENTS Interpreter Development)
FetchContent_MakeAvailable(PyBind11)
message(STATUS "Creating binding for module ${name}")
file(GLOB_RECURSE pybind_src_files "python_binding/*.cpp")
pybind11_add_module(${name} MODULE ${pybind_src_files} "NO_EXTRAS") # NO EXTRA recquired for pip install
target_include_directories(${name} PUBLIC "python_binding")
target_link_libraries(${name} PUBLIC ${target_to_bind})
endfunction()
include(${CMAKE_CURRENT_LIST_DIR}/aidge_backend_cpu-config-version.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/aidge_backend_cpu-targets.cmake)
#ifndef DNN_HPP
#define DNN_HPP
#include <aidge/graph/GraphView.hpp>
/**
* @brief This file contains all of what is related to the construction of the
* neural network
*
*/
/**
* @brief This function generate the exported Aidge::GraphView.
*
* @return std::shared_ptr<Aidge::GraphView>
*/
std::shared_ptr<Aidge::GraphView> generateModel();
#endif /* DNN_HPP */
#include <iostream>
#include <aidge/backend/cpu.hpp>
#include "include/dnn.hpp"
int main()
{
std::cout << "BEGIN" << std::endl;
std::shared_ptr<Aidge::GraphView> graph = generateModel();
std::cout << "END" << std::endl;
return 0;
}
export
#include <pybind11/pybind11.h>
#include "dnn.hpp"
namespace py = pybind11;
void init_export(py::module& m){
m.def("generate_model", generateModel);
}
PYBIND11_MODULE(export, m) {
init_export(m);
}
0.0.0
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment