Merge branch 'main' into fuseBN

.gitlab/ci/coverage.gitlab-ci.yml

@@ -24,8 +24,10 @@ coverage:ubuntu_python:
   script:
     - source venv/bin/activate
     - python3 -m pip install numpy coverage
-    - cd aidge_core
-    - python3 -m coverage run --source=. -m unittest discover -s unit_tests/ -v -b
+    - cd ${CI_PROJECT_NAME}
+    # Retrieve the installation path of the module, since it is installed with pip.
+    - export MODULE_LOCATION=`python -c "import ${CI_PROJECT_NAME} as _; print(_.__path__[0])"`
+    - python3 -m coverage run --source=$MODULE_LOCATION -m unittest discover -s unit_tests/ -v -b
     - python3 -m coverage report
     - python3 -m coverage xml
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
@@ -33,4 +35,4 @@ coverage:ubuntu_python:
     reports:
       coverage_report:
         coverage_format: cobertura
-        path: aidge_core/coverage.xml
+        path: ${CI_PROJECT_NAME}/coverage.xml

.gitlab/ci/static_analysis.gitlab-ci.yml

@@ -26,8 +26,8 @@ static_analysis:python:
   script:
     - pip install pylint
     - pip install pylint-gitlab
-    - pylint --rcfile=.pylintrc --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter aidge_core/ > codeclimate.json
-    - pylint --rcfile=.pylintrc --exit-zero --output-format=pylint_gitlab.GitlabPagesHtmlReporter aidge_core/ > pylint.html
+    - pylint --rcfile=.pylintrc --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter ${CI_PROJECT_NAME}/ > codeclimate.json
+    - pylint --rcfile=.pylintrc --exit-zero --output-format=pylint_gitlab.GitlabPagesHtmlReporter ${CI_PROJECT_NAME}/ > pylint.html
     - mkdir -p public/python/$CI_COMMIT_REF_NAME
     - mv pylint.html public/python/$CI_COMMIT_REF_NAME/
   artifacts:

.gitlab/ci/test.gitlab-ci.yml

@@ -17,14 +17,14 @@ test:ubuntu_python:
     - docker
   script:
     - source venv/bin/activate
-    - cd aidge_core
+    - cd ${CI_PROJECT_NAME}
     - python3 -m pip install unittest-xml-reporting
     - python3 -m pip list
     # Run on discovery all tests located in core/unit_tests/python
     - python3 -m xmlrunner discover -s unit_tests/ -v -b --output-file xmlrunner-results.xml
   artifacts:
     reports:
-      junit: aidge_core/xmlrunner-results.xml
+      junit: ${CI_PROJECT_NAME}/xmlrunner-results.xml
 
 test:windows_cpp:
   stage: test

include/aidge/backend/OperatorImpl.hpp

@@ -20,7 +20,7 @@ namespace Aidge {
 class OperatorImpl {
 public:
     virtual void forward(){};
-    virtual void backward() {}
+    virtual void backward(){};
 
     /**
      * @brief Minimum amount of data from a specific input required by the
@@ -46,13 +46,19 @@ public:
     virtual NbElts_t getNbConsumedData(const IOIndex_t inputIdx) const = 0;
 
     /**
-     * @brief TOtal amount of produced data ready to be used on a specific output.
+     * @brief Total amount of produced data ready to be used on a specific output.
      *
      * @param outputIdx Index of the output analysed.
      * @return DimSize_t
      */
     virtual NbElts_t getNbProducedData(const IOIndex_t outputIdx) const = 0;
 
+    /**
+     * @brief Update the Consummer Producer system by simulating the consumption and production of i/o
+     *
+     */
+    virtual void updateConsummerProducer() = 0;
+
     virtual ~OperatorImpl() = default;
 };
 } // namespace Aidge

include/aidge/graph/GraphView.hpp

@@ -208,7 +208,7 @@ public:
      * @brief Get the Nodes pointed to by the GraphView object.
      * @return std::set<NodePtr>
      */
-    inline std::set<NodePtr> getNodes() const { return mNodes; }
+    inline const std::set<NodePtr>& getNodes() const { return mNodes; }
 
     /**
      * @brief Get the operator with the corresponding name if it is in the

include/aidge/graph/OpArgs.hpp

@@ -55,7 +55,7 @@ public:
  * @param inputs List of Node and GraphView to link sequentially.
  * @return std::shared_ptr<GraphView> Pointer to the generated view.
  */
-std::shared_ptr<GraphView> Sequential(std::initializer_list<OpArgs> inputs);
+std::shared_ptr<GraphView> Sequential(std::vector<OpArgs> inputs);
 
 /////////////////////////////
 // Parallel
@@ -65,7 +65,7 @@ std::shared_ptr<GraphView> Sequential(std::initializer_list<OpArgs> inputs);
  * @param inputs List of Node and GraphView to link sequentially.
  * @return std::shared_ptr<GraphView> pointer to the generated view.
  */
-std::shared_ptr<GraphView> Parallel(std::initializer_list<OpArgs> inputs);
+std::shared_ptr<GraphView> Parallel(std::vector<OpArgs> inputs);
 
 /////////////////////////////
 // Residual
@@ -79,8 +79,8 @@ std::shared_ptr<GraphView> Parallel(std::initializer_list<OpArgs> inputs);
  * @param inputs List of Node and GraphView to link sequentially.
  * @return std::shared_ptr<GraphView> pointer to the generated view.
  */
-std::shared_ptr<GraphView> Residual(std::initializer_list<OpArgs> inputs);
+std::shared_ptr<GraphView> Residual(std::vector<OpArgs> inputs);
 
 }
 
-#endif /* AIDGE_CORE_GRAPH_OPARGS_H_ */
\ No newline at end of file
+#endif /* AIDGE_CORE_GRAPH_OPARGS_H_ */

include/aidge/operator/Operator.hpp

@@ -78,6 +78,8 @@ public:
      */
     NbElts_t getNbProducedData(const IOIndex_t outputIdx) const;
 
+    void updateConsummerProducer();
+
     virtual void forward();
 
     virtual void backward();

include/aidge/scheduler/Scheduler.hpp

@@ -43,6 +43,8 @@ public:
     };
     ~SequentialScheduler() = default;
 
+    void generateScheduling(bool verbose = false);
+
     /**
      * @brief Run the provided Computational Graph with a batch of data
      */
@@ -54,6 +56,15 @@ public:
      */
     void saveSchedulingDiagram(const std::string& fileName) const;
 
+    /**
+     * @brief Return a vector of Node ordered by the order they are called by the scheduler
+     *
+     * @return std::vector<std::shared_ptr<Node>>
+     */
+    std::vector<std::shared_ptr<Node>> getStaticScheduling(){
+        return mStaticSchedule;
+    }
+
 private:
     /**
      * @brief Set of layers receiving an input from currently processing layers
@@ -63,9 +74,27 @@ private:
      */
     std::set<std::shared_ptr<Node>> getConsumers(const std::set<std::shared_ptr<Node>>& producers) const;
 
+    /**
+     * @brief Shared ptr to the scheduled graph view
+     *
+     */
     std::shared_ptr<GraphView> mGraphView;
+    /**
+     * @brief List of SchedulingElement (i.e: Nodes with their computation time)
+     *
+     */
     std::vector<SchedulingElement> mScheduling;
+    /**
+     * @brief List of nodes ordered by their
+     *
+     */
+    std::vector<std::shared_ptr<Node>> mStaticSchedule;
+    /**
+     * @brief Number of computation node (i.e: nb nodes != Producer)
+     *
+     */
+    std::size_t mComputationNumber = 0; // TODO: Check if not inferable from mStaticSchedule
 };
 } // namespace Aidge
 
-#endif /* AIDGE_SCHEDULER_H_ */
\ No newline at end of file
+#endif /* AIDGE_SCHEDULER_H_ */

python_binding/graph/pybind_OpArgs.cpp

@@ -10,19 +10,20 @@
  ********************************************************************************/
 
 #include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
 #include "aidge/graph/OpArgs.hpp"
 #include "aidge/graph/Node.hpp"
 #include "aidge/graph/GraphView.hpp"
-#include <pybind11/stl.h>
-#include <pybind11/complex.h>
-#include <pybind11/functional.h>
-#include <pybind11/chrono.h>
+
 
 
 namespace py = pybind11;
 namespace Aidge {
 void init_OpArgs(py::module& m){
     py::class_<OpArgs, std::shared_ptr<OpArgs>>(m, "OpArgs")
+    .def(py::init<const std::shared_ptr<GraphView>&>(), py::arg("view_"))
+    .def(py::init<const std::shared_ptr<Node>&>(), py::arg("node_"))
     .def("node", &OpArgs::node)
     .def("view", &OpArgs::view)
     ;

python_binding/operator/pybind_GenericOperator.cpp

@@ -22,7 +22,7 @@ namespace Aidge {
 void init_GenericOperator(py::module& m) {
     py::class_<GenericOperator_Op, std::shared_ptr<GenericOperator_Op>, Operator>(m, "GenericOperatorOp",
                                                                                   py::multiple_inheritance())
-            .def("get_parameter_type", &GenericOperator_Op::getParameterType)
+    .def("get_parameter_type", &GenericOperator_Op::getParameterType)
     .def("get_parameters_name", &GenericOperator_Op::getParametersName)
     .def("add_parameter", &GenericOperator_Op::addParameter<bool>)
     .def("add_parameter", &GenericOperator_Op::addParameter<int>)
@@ -34,10 +34,10 @@ void init_GenericOperator(py::module& m) {
     .def("add_parameter", &GenericOperator_Op::addParameter<std::vector<std::string>>)
     .def("get_parameter", [](GenericOperator_Op& self, std::string key) -> py::object {
         /*
-        This getParameter method returns the good python type without having to have 
+        This getParameter method returns the good python type without having to have
         prior knowledge of the parameter type.
         */
-        py::object res = py::none(); 
+        py::object res = py::none();
         std::string paramType = self.getParameterType(key);
         if(paramType == typeid(int).name())
             res = py::cast(self.getParameter<int>(key));

python_binding/scheduler/pybind_Scheduler.cpp

@@ -10,6 +10,7 @@
  ********************************************************************************/
 
 #include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
 #include "aidge/scheduler/Scheduler.hpp"
 #include "aidge/graph/GraphView.hpp"
 
@@ -20,6 +21,8 @@ void init_Scheduler(py::module& m){
     .def(py::init<std::shared_ptr<GraphView>&>(), py::arg("graph_view"))
     .def("forward", &SequentialScheduler::forward, py::arg("forward_dims")=true, py::arg("verbose")=false)
     .def("save_scheduling_diagram", &SequentialScheduler::saveSchedulingDiagram, py::arg("file_name"))
+    .def("generate_scheduling", &SequentialScheduler::generateScheduling, py::arg("verbose")=false)
+    .def("get_static_scheduling", &SequentialScheduler::getStaticScheduling)
     ;
 }
 }

setup.ps1

+# Helper setup tool to automatically build aidge_core on Windows.
+
+# Requirements
+################################################################################
+# You have either VS BuildTools or VS Community already present on your 
+# system, with the build tools installed.
+# If not, download Visual Studio Community here:
+# https://visualstudio.microsoft.com/fr/vs/community/
+# Make sure to install the "Desktop Development with C++" workload.
+# Run this script in a Powershell console with Administrator rights in order to
+# automatically install the dependencies, or just execute the second part if you
+# already have all the dependencies satisfied.
+
+# Enable or disable automatic installation of requirements
+# Run .\setup.ps1 -install_reqs:$false to disable it
+param ([bool]$install_reqs=$true)
+
+# Default install path is .\install_cpp
+if (-not $env:AIDGE_INSTALL_PATH)
+{
+    $env:AIDGE_INSTALL_PATH = $(Join-Path $pwd install_cpp)
+}
+
+# 1. Setup environment
+################################################################################
+if ($install_reqs)
+{
+    # Install Chocolatey
+    Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+    # Install dependencies
+    choco install cmake.install --installargs '"ADD_CMAKE_TO_PATH=System"' -Y
+    choco install git -Y
+    choco install python -Y
+    # Update PATH
+    $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+}
+
+# 2. Compile & install aidge_core
+################################################################################
+mkdir -Force build_cpp
+mkdir -Force $env:AIDGE_INSTALL_PATH
+Set-Location build_cpp
+cmake -DCMAKE_INSTALL_PREFIX:PATH=$env:AIDGE_INSTALL_PATH -DCMAKE_BUILD_TYPE=Debug ..
+if(!$?) { $lastError = $LASTEXITCODE; Set-Location $PSScriptRoot; Exit $lastError }
+cmake --build . -j2
+if(!$?) { $lastError = $LASTEXITCODE; Set-Location $PSScriptRoot; Exit $lastError }
+cmake --install . --config Debug
+if(!$?) { $lastError = $LASTEXITCODE; Set-Location $PSScriptRoot; Exit $lastError }
+# Optional: run the unit tests
+ctest --output-on-failure
+if(!$?) { $lastError = $LASTEXITCODE; Set-Location $PSScriptRoot; Exit $lastError }
+Set-Location $PSScriptRoot

src/graph/Connector.cpp

@@ -39,7 +39,7 @@ std::shared_ptr<Aidge::GraphView> Aidge::generateGraph(std::vector<Connector> ct
             graph->add(nodesToAdd.back());  // only add, connection already done
                                             // between nodes
             std::vector<std::shared_ptr<Node>> parents = nodesToAdd.back()->getParents();
-            std::set<std::shared_ptr<Node>> alreadyAdded = graph->getNodes();
+            const std::set<std::shared_ptr<Node>>& alreadyAdded = graph->getNodes();
             for (std::shared_ptr<Node> parent : parents) {
                 if (alreadyAdded.find(parent) == alreadyAdded.end()) {
                     buffer.push_back(parent);

src/graph/OpArgs.cpp

@@ -14,13 +14,13 @@
 #include "aidge/graph/OpArgs.hpp"
 
 
-std::shared_ptr<Aidge::GraphView> Aidge::Sequential(std::initializer_list<OpArgs> inputs) {
+std::shared_ptr<Aidge::GraphView> Aidge::Sequential(std::vector<OpArgs> inputs) {
     std::shared_ptr<GraphView> gv = std::make_shared<GraphView>();
     for (const OpArgs& elt : inputs) {
         if(elt.node() != nullptr) {
             // >= to allow incomplete graphViews
             assert(static_cast<std::size_t>(elt.node()->getNbFreeDataInputs()) >= gv->outputNodes().size());
-            /* 
+            /*
             *  /!\ mn.view()->outputNodes() is a set, order of Nodes cannot be guaranted.
             *  Prefer a functional description for detailed inputs
             */
@@ -44,7 +44,7 @@ std::shared_ptr<Aidge::GraphView> Aidge::Sequential(std::initializer_list<OpArgs
 }
 
 
-std::shared_ptr<Aidge::GraphView> Aidge::Parallel(std::initializer_list<OpArgs> inputs) {
+std::shared_ptr<Aidge::GraphView> Aidge::Parallel(std::vector<OpArgs> inputs) {
     std::shared_ptr<GraphView> gv = std::make_shared<GraphView>();
     for(const OpArgs& elt : inputs) {
         if (elt.node()!=nullptr)
@@ -56,7 +56,7 @@ std::shared_ptr<Aidge::GraphView> Aidge::Parallel(std::initializer_list<OpArgs>
 }
 
 
-std::shared_ptr<Aidge::GraphView> Aidge::Residual(std::initializer_list<OpArgs> inputs) {
+std::shared_ptr<Aidge::GraphView> Aidge::Residual(std::vector<OpArgs> inputs) {
     std::shared_ptr<GraphView> gv = Sequential(inputs);
     assert(gv->outputNodes().size() == 1U && "Zero or more than one output Node for the GraphView, don't know which one to choose from for the residual connection");
     std::shared_ptr<Node> lastNode = *gv->outputNodes().begin();
@@ -70,4 +70,4 @@ std::shared_ptr<Aidge::GraphView> Aidge::Residual(std::initializer_list<OpArgs>
     assert(lastNode->getNbFreeDataInputs()>=1);
     gv->addChild(lastNode, firstNode, 0U, gk_IODefaultIndex);
     return gv;
-}
\ No newline at end of file
+}

src/operator/Operator.cpp

@@ -38,6 +38,9 @@ Aidge::NbElts_t Aidge::Operator::getNbConsumedData(Aidge::IOIndex_t inputIdx) co
 Aidge::NbElts_t Aidge::Operator::getNbProducedData(Aidge::IOIndex_t outputIdx) const {
     return mImpl->getNbProducedData(outputIdx);
 }
+void Aidge::Operator::updateConsummerProducer(){
+    mImpl->updateConsummerProducer();
+}
 
 void Aidge::Operator::forward() { mImpl->forward(); }
 

src/scheduler/Scheduler.cpp

@@ -33,26 +33,19 @@ void drawProgressBar(double progress, int barWidth, const std::string& additiona
     fflush(stdout);
 }
 
-// TODO: handle multiple inputs/outputs
-void Aidge::SequentialScheduler::forward(bool frowardDims, bool verbose) {
-    if (frowardDims) {mGraphView->forwardDims(); }
-
-    mScheduling.clear();
-
+void Aidge::SequentialScheduler::generateScheduling(bool verbose) {
     // setup initial producers list
-    // add each Producer Node.
-    std::set<std::shared_ptr<Node>> computationOver;
-    std::size_t computationNumber = 0;
+    mComputationNumber = 0;
     std::set<std::shared_ptr<Node>> producers;
     for (const std::shared_ptr<Node>& nodePtr : mGraphView->getNodes()) {
         if (nodePtr->type() == "Producer") {
             producers.insert(nodePtr);
         } else {
-            ++computationNumber;
+            ++mComputationNumber;
         }
     }
     // add Data Input
-    // FIXME : shoudl be changed when the real system for providing
+    // FIXME : should be changed when the real system for providing
     // data is implemented
     for (const std::shared_ptr<Node>& nodePtr : mGraphView->inputNodes()) {
         for (const auto& parentPtr : nodePtr->getParents()) {
@@ -112,21 +105,10 @@ void Aidge::SequentialScheduler::forward(bool frowardDims, bool verbose) {
             }
         }
 
-        // run sequencially every runnable consumers once
-        // TODO: handle memory allocation in scheduler
-        // TODO: optimize memory usage
+        // Push consumers in the list of nodes to run and update the consumer producer system
         for (const auto& runnable : runnableConsumers) {
-            if (verbose)
-                printf("run: %s\n",
-                       (runnable->type() + "_" + std::to_string(reinterpret_cast<uintptr_t>(runnable.get()))).c_str());
-            else
-                drawProgressBar(static_cast<float>(computationOver.size()) / static_cast<float>(computationNumber), 50,
-                                (std::string("running ") + runnable->type() + "_" +
-                                 std::to_string(reinterpret_cast<uintptr_t>(runnable.get()))));
-            const auto tStart = std::chrono::high_resolution_clock::now();
-            runnable->forward();
-            const auto tEnd = std::chrono::high_resolution_clock::now();
-            mScheduling.push_back(SchedulingElement(runnable, tStart, tEnd));
+            runnable->getOperator()->updateConsummerProducer();
+            mStaticSchedule.push_back(runnable);
         }
 
         // update producers and consumers list
@@ -164,18 +146,6 @@ void Aidge::SequentialScheduler::forward(bool frowardDims, bool verbose) {
                 }
             }
 
-            bool computationOverForConsumer = true;
-            for (IOIndex_t parentIDi = 0; parentIDi < consumer->nbInputs(); ++parentIDi) {
-                if (consumer->getOperator()->getNbConsumedData(parentIDi) <
-                    consumer->getOperator()->getNbRequiredData(parentIDi)) {
-                    computationOverForConsumer = false;
-                    break;
-                }
-            }
-            if (computationOverForConsumer) {
-                computationOver.insert(consumer);
-            }
-
             for (IOIndex_t outId = 0; outId < consumer->nbOutputs(); ++outId) {
                 if (consumer->getOperator()->getNbProducedData(outId) > 0) {
                     if (verbose) printf("  also producer\n");
@@ -197,8 +167,52 @@ void Aidge::SequentialScheduler::forward(bool frowardDims, bool verbose) {
 
         if (verbose) printf("*************\n");
     } while (!consumers.empty());
+
+}
+
+// TODO: handle multiple inputs/outputs
+void Aidge::SequentialScheduler::forward(bool forwardDims, bool verbose) {
+    if (forwardDims) {mGraphView->forwardDims(); }
+
+    // add each Producer Node.
+    std::set<std::shared_ptr<Node>> computationOver;
+
+    mScheduling.clear();
+
+    this->generateScheduling();
+
+    // TODO: For loop on the list of node to run
+    // run sequencially every runnable consumers once
+    // TODO: handle memory allocation in scheduler
+    // TODO: optimize memory usage
+    for (const auto& runnable : mStaticSchedule) {
+        bool computationOverForConsumer = true;
+        for (IOIndex_t parentIDi = 0; parentIDi < runnable->nbInputs(); ++parentIDi) {
+            if (runnable->getOperator()->getNbConsumedData(parentIDi) <
+                runnable->getOperator()->getNbRequiredData(parentIDi)) {
+                computationOverForConsumer = false;
+                break;
+            }
+        }
+        if (computationOverForConsumer) {
+            computationOver.insert(runnable);
+        }
+
+        if (verbose)
+            printf("run: %s\n",
+                    (runnable->type() + "_" + std::to_string(reinterpret_cast<uintptr_t>(runnable.get()))).c_str());
+        else
+            drawProgressBar(static_cast<float>(computationOver.size()) / static_cast<float>(mComputationNumber), 50,
+                            (std::string("running ") + runnable->type() + "_" +
+                                std::to_string(reinterpret_cast<uintptr_t>(runnable.get()))));
+        const auto tStart = std::chrono::high_resolution_clock::now();
+        runnable->forward();
+        const auto tEnd = std::chrono::high_resolution_clock::now();
+        mScheduling.push_back(SchedulingElement(runnable, tStart, tEnd));
+    }
     if (!verbose) drawProgressBar(1.0, 50, "                                   ");
     printf("\n");
+
 }
 
 void Aidge::SequentialScheduler::saveSchedulingDiagram(const std::string& fileName) const {
@@ -231,4 +245,4 @@ std::set<std::shared_ptr<Aidge::Node>> Aidge::SequentialScheduler::getConsumers(
     }
 
     return consumers;
-}
\ No newline at end of file
+}