Merge branch 'user/cguillon/dev/speedup-test-export' into 'dev'

[Unit Tests]  Speedup python unit tests and Add some foundation for factorized aidge_core.testing module between components

See merge request eclipse/aidge/aidge_core!173

aidge_core/aidge_export_aidge/export.py

@@ -54,7 +54,7 @@ def export(export_folder: str,
     ### Generating an export for each nodes and dnn file ###
     list_configs = []  # List of headers to include in dnn.cpp to access attribute and parameters
     list_actions = []  # List of string to construct graph
-    set_operator = set()
+    list_operators = [] # List of operator types used (to be made unique latter)
     # Queue of Aidge nodes to explore, guarantee a topological exploration of the graph
     open_nodes = list(graph_view.get_input_nodes())
     # List of Aidge nodes already explored
@@ -82,7 +82,7 @@ def export(export_folder: str,
         open_nodes += list(node.get_children())
 
         if node.type() in supported_operators():
-            set_operator.add(node.type())
+            list_operators.append(node.type())
             op = OPERATORS_REGISTRY[node.type()](node)
 
             # TODO: list_configs and list_actions don't need to be passed by argument
@@ -94,11 +94,13 @@ def export(export_folder: str,
         else:
             raise RuntimeError(f"Operator: {node.type()} is not supported")
         closed_nodes.append(node)
+    list_operators = list(dict.fromkeys(list_operators)) # make unique
+
     # Generate full dnn.cpp
     aidge_core.generate_file(
         export_folder_path / "src/dnn.cpp",
         ROOT_EXPORT / "templates/dnn.jinja",
         headers=list_configs,
-        operators=set_operator,
+        operators=list_operators,
         actions=list_actions,
     )

aidge_core/testing/__init__.py

+#
+# Do not add there auto import of submodules.
+#
+# The testing module contains utils and other tools
+# related to tests, possibly reusable by other aidge
+# components unit_tests.
+#
+# Import a specific module explicitly with for instance:
+# import aidge_core.testing.utils
+# or
+# from aidge_core.testing.utils import (....,)
+#

aidge_core/testing/utils/__init__.py

+#
+# Should provide some general utility functions for testing.
+# For instance:
+# - filesystem
+# - os dependencies
+# - unit tests setup
+#
+
+from .tree_cache import tree_update_from_cache
+from .tree_utils import tree_move, tree_remove

aidge_core/testing/utils/tree_cache.py

+"""
+
+Provide tree_update_from_cache(path) method which
+minimize changes in a generated tree when files are
+re-generated but identical.
+
+It takes as argument a generated tree, and optionally a cache path.
+Then it will update both the generated tree and the cache tree
+to take the cache version of the files when identical, or the newly
+generated one otherwise.
+
+This is in particular useful for speeding up iterative compilation
+when generating a source/build system tree.
+
+For instance:
+- first time, one generates a tree of files:
+  - generated: path/{t1,t2,t3}
+- then call tree_update_from_cache("path")
+  - will generate: __cache_path/{t1,t2,t3}
+  - and untouch: path/{t1,t2,t3}
+- second time, re-generate a tree of file:
+  - say generated files are identical: path/{t1,t2,t3}
+- then call tree_update_from_cache("path")
+  - will untouch in cache: __cache_path/{t1,t2,t3}
+  - and reset to previous timestamps files: path/{t1,t2,t3}
+- third time, re-generate again with some changes:
+  - say t1 is identical, t2 content has changed and no t3: path/{t1,t2'}
+- then call tree_update_from_cache("path")
+  - will update t2' and remove t3 in cache: __cache_path/{t1,t2'}
+  - and reset to previous timestamp t1: path/{t1,t2'}
+
+Note that by default the `dir`/__cache_`name` cache path is used
+for a given path `dir`/`name`.
+Though it is also possible to have the cache path inside the generated tree,
+in this case use for instance:
+
+    tree_update_from_cache(path, Path(path) / "__cache_src")
+
+For more evolved scenarii, specialize the provided FileTreeCache class.
+
+"""
+
+
+from pathlib import Path
+import shutil
+import filecmp
+from typing import Optional, Union, List
+
+from .tree_utils import tree_move, tree_remove
+
+
+__all__ = [
+    "FileTreeCache",
+    "tree_update_from_cache",
+]
+
+
+class FileTreeCache():
+    """
+    Class for implementation of the file tree cache.
+    Can be derived to changes for instance default cache name/tmp name prefixes
+    or to specialize for other contexts.
+    """
+    default_cache_prefix = "__cache_"
+    default_tmp_cache_prefix = "__tmp_cache_"
+    default_tmp_prefix = "__tmp_"
+
+    def __init__(self,
+                 src_path: Union[str|Path],
+                 cache_path: Optional[Union[str|Path]] = None
+                 ) -> None:
+        self.src_path = Path(src_path).absolute()
+        self.cache_path = (
+            Path(cache_path).absolute()
+            if cache_path is not None else
+            (self.src_path.parent /
+             f"{self.default_cache_prefix}{self.src_path.name}")
+        )
+        ctx_msg = f"tree_cache: {src_path = }, {cache_path = }"
+        assert self.src_path != self.cache_path, f"src_path and cache_path must differ on {ctx_msg}"
+        assert not self.src_path.is_relative_to(self.cache_path), f"src_path must not be relative to cache_path on {ctx_msg}"
+        self._tmp_path = (
+            self.src_path.parent /
+            f"{self.default_tmp_prefix}{self.src_path.name}")
+        self._tmp_cache_path = (
+            self.src_path.parent /
+            f"{self.default_tmp_cache_prefix}{self.src_path.name}")
+
+    @classmethod
+    def _copytree_or_cache(cls, src_dir: Path, dst_dir: Path, cache_dir: Path, dst_cache_dir: Path) -> None:
+        assert not dst_dir.exists()
+        assert not dst_cache_dir.exists()
+        assert src_dir.is_dir()
+        assert not cache_dir.exists() or cache_dir.is_dir()
+        assert not cache_dir.is_relative_to(src_dir)
+
+        def copy_or_cache(src, dst):
+            base_src = Path(src).relative_to(src_dir)
+            cache_src = cache_dir / base_src
+            base_dst = Path(dst).relative_to(dst_dir)
+            cache_dst = dst_cache_dir / base_dst
+            cache_dst.parent.mkdir(parents=True, exist_ok=True)
+            if cache_src.exists() and filecmp.cmp(str(src), str(cache_src), shallow=False):
+                shutil.copy2(str(cache_src), str(cache_dst))
+                shutil.copy2(str(cache_src), dst)
+            else:
+                shutil.copy2(src, str(cache_dst))
+                shutil.copy2(src, dst)
+        shutil.copytree(str(src_dir), str(dst_dir), copy_function=copy_or_cache)
+
+    def update_from_cache(self) -> None:
+        assert self.src_path.exists(), f"src path must exist before swapping with cache"
+
+        # Move cache path apart first as it may be relative to source path
+        tree_move(self.cache_path, self._tmp_cache_path, ignore_missing=True, exist_ok=True)
+        # Move source path apart before recreating merged source tree
+        tree_move(self.src_path, self._tmp_path, exist_ok=True)
+
+        # Manage the source/cache merge to the dst/dst_cahe with a variant of
+        # copytree.
+        self._copytree_or_cache(
+            src_dir=self._tmp_path,
+            dst_dir=self.src_path,
+            cache_dir=self._tmp_cache_path,
+            dst_cache_dir=self.cache_path,
+        )
+
+        # Remove tmp source path
+        tree_remove(self._tmp_path)
+        # Note that the tmp cache path may not exist
+        tree_remove(self._tmp_cache_path, ignore_missing=True)
+
+
+def tree_update_from_cache(
+        src_path: Union[str|Path],
+        cache_path: Optional[Union[str|Path]] = None) -> None:
+    """
+    Update from cache the current generation of a tree from the
+    older generations, preserving file stamps when files contents are identical.
+
+    :param src_path: str or Path object to the generated tree
+    :param cache_path: optional str or Path object to the cache path,
+    or defaults to: `cache_path = src_path.parent / f"__cache_{src_path.name}"`
+    """
+    FileTreeCache(src_path, cache_path).update_from_cache()

aidge_core/testing/utils/tree_utils.py

+"""
+
+Provide utility function for file trees manipulations.
+
+"""
+
+import shutil
+from pathlib import Path
+from typing import Union, Optional
+
+
+__all__ = [
+    "tree_move",
+    "tree_remove",
+]
+
+
+def tree_remove(
+        path: Union[str|Path],
+        ignore_missing: bool = False,
+) -> None:
+    """
+    Remove the full tree at path.
+    Optionally ignore if the path does not exist when ignore_missing is True.
+
+    :param path: str or Path object to the directory path
+    :param ignore_missing: if True will return early is path does not exists
+    """
+    path = Path(path)
+    ctx_msg = f"tree_remove: : {path = }"
+    assert ignore_missing or path.exists(), f"path must exists when ignore_missing is False on {ctx_msg}"
+    if ignore_missing and not path.exists():
+        return
+    shutil.rmtree(path)
+
+
+def tree_move(
+        src_path: Union[str|Path],
+        dst_path: Union[str|Path],
+        ignore_missing: bool = False,
+        exist_ok: bool = False,
+) -> None:
+    """
+    Move the whole src_path file tree to dst_path.
+    Optionally does nothing if the src path does not exists and ignore_missing is True.
+    Optionally the full dst_path will be removed first when exists_ok is True.
+
+    :param src_path: str or Path object to the source directory path
+    :param dst_path: str or Path object to the new path name for the source directory
+    :param ignore_missing: if True will return early is src_path does not exists
+    :param exist_ok: if True will first erase the new path name if it exists
+    """
+    src_path = Path(src_path)
+    dst_path = Path(dst_path)
+    ctx_msg = f"tree_move: : {src_path = }, {dst_path = }"
+    assert ignore_missing or src_path.exists(), f"src_path must exists when ignore_missing is False on {ctx_msg}"
+    assert exist_ok or not dst_path.exists(), f"dst_path must not exists when exist_ok is False on {ctx_msg}"
+    assert src_path != dst_path, f"paths must not be identical on {ctx_msg}"
+    assert not dst_path.is_relative_to(src_path), f"dst_path must not be relative to src_path on {ctx_msg}"
+    assert not src_path.is_relative_to(dst_path), f"src_path must not be relative to dst_path on {ctx_msg}"
+    if ignore_missing and not src_path.exists():
+        return
+    if exist_ok and dst_path.exists():
+        shutil.rmtree(dst_path)
+    shutil.move(src_path, dst_path)

aidge_core/unit_tests/test_export.py

@@ -8,8 +8,6 @@ http://www.eclipse.org/legal/epl-2.0.
 SPDX-License-Identifier: EPL-2.0
 """
 
-import aidge_core
-from aidge_core.utils import run_command
 import unittest
 import os
 import pathlib
@@ -18,6 +16,10 @@ import subprocess
 import sys
 
 
+import aidge_core
+from aidge_core.utils import run_command
+from aidge_core.testing.utils import tree_update_from_cache, tree_move, tree_remove
+
 def initFiller(model):
     # Initialize parameters (weights and biases)
     for node in model.get_nodes():
@@ -26,6 +28,8 @@ def initFiller(model):
             value = prod_op.get_output(0)
             value.set_backend("cpu")
             tuple_out = node.output(0)[0]
+            # Force seed before filler for reproducibility
+            aidge_core.random.Generator.set_seed(0)
             # No conv in current network
             if tuple_out[0].type() == "Conv" and tuple_out[1] == 1:
                 # Conv weight
@@ -43,22 +47,6 @@ def initFiller(model):
                 pass
 
 
-def clean_dir(dir: pathlib.Path) -> None:
-    if not dir.is_dir():
-        print(f"Error : directory {dir} doesn't exist. Exiting clean_dir().")
-        return
-    for filename in os.listdir(dir):
-        file_path = os.path.join(dir, filename)
-        try:
-            if os.path.isfile(file_path) or os.path.islink(file_path):
-                os.unlink(file_path)
-            elif os.path.isdir(file_path):
-                shutil.rmtree(file_path)
-        except Exception as e:
-            print(f"Failed to delete {file_path}. Reason: {e}")
-    return
-
-
 class test_export(unittest.TestCase):
     """Test aidge export"""
 
@@ -66,6 +54,10 @@ class test_export(unittest.TestCase):
         self.EXPORT_PATH: pathlib.Path = pathlib.Path("dummy_export")
         self.BUILD_DIR: pathlib.Path = self.EXPORT_PATH / "build"
         self.INSTALL_DIR: pathlib.Path = (self.EXPORT_PATH / "install").absolute()
+        self.TMP_BUILD_DIR: pathlib.Path = (
+            self.EXPORT_PATH.parent /
+            f"__tmp_{self.EXPORT_PATH.name}_build"
+        )
 
     def tearDown(self):
         pass
@@ -76,28 +68,40 @@ class test_export(unittest.TestCase):
         model = aidge_core.sequential(
             [
                 aidge_core.FC(
-                    in_channels=32 * 32 * 3, out_channels=512, name="InputNode"
+                    in_channels=32 * 32 * 3, out_channels=64, name="InputNode"
                 ),
                 aidge_core.ReLU(name="Relu0"),
-                aidge_core.FC(in_channels=512, out_channels=256, name="FC1"),
+                aidge_core.FC(in_channels=64, out_channels=32, name="FC1"),
                 aidge_core.ReLU(name="Relu1"),
-                aidge_core.FC(in_channels=256, out_channels=128, name="FC2"),
+                aidge_core.FC(in_channels=32, out_channels=16, name="FC2"),
                 aidge_core.ReLU(name="Relu2"),
-                aidge_core.FC(in_channels=128, out_channels=10, name="OutputNode"),
+                aidge_core.FC(in_channels=16, out_channels=10, name="OutputNode"),
             ]
         )
 
         initFiller(model)
 
+        # Preserve previously generated build if present
+        tree_move(self.BUILD_DIR, self.TMP_BUILD_DIR, ignore_missing=True, exist_ok=True)
+        # Clean install dir
+        tree_remove(self.INSTALL_DIR, ignore_missing=True)
+
         # Export model
         aidge_core.export(self.EXPORT_PATH, model)
-
-        self.assertTrue(
-            self.EXPORT_PATH.is_dir(), "Export folder has not been generated"
+        self.assertTrue(self.EXPORT_PATH.is_dir(), "Export folder has not been generated")
+        # Add other source files
+        shutil.copyfile(pathlib.Path(__file__).parent / "static/main.cpp", self.EXPORT_PATH / "main.cpp")
+
+        # Use cache if any, put cache inside export dir
+        # such that cleaning export dir also cleans the cache
+        tree_update_from_cache(
+            self.EXPORT_PATH,
+            cache_path=self.EXPORT_PATH / "__cache_export"
         )
-        os.makedirs(self.BUILD_DIR, exist_ok=True)
-        clean_dir(self.BUILD_DIR)  # if build dir existed already ensure its emptyness
-        clean_dir(self.INSTALL_DIR)
+
+        # Move back preserved build dir if any and ensure build dir exists
+        tree_move(self.TMP_BUILD_DIR, self.BUILD_DIR, ignore_missing=True)
+        self.BUILD_DIR.mkdir(exist_ok=True)
 
         # Test compilation of export
         search_path = (
@@ -106,11 +110,6 @@ class test_export(unittest.TestCase):
             else os.environ["AIDGE_INSTALL"]
         )
 
-        shutil.copyfile(
-            pathlib.Path(__file__).parent / "static/main.cpp",
-            self.EXPORT_PATH / "main.cpp",
-        )
-
         ##########################
         # CMAKE EXPORT
         try: