aidge_core/testing/utils/tree_cache.py

+"""
+
+Provide tree_update_from_cache(path) method which
+minimize changes in a generated tree when files are
+re-generated but identical.
+
+It takes as argument a generated tree, and optionally a cache path.
+Then it will update both the generated tree and the cache tree
+to take the cache version of the files when identical, or the newly
+generated one otherwise.
+
+This is in particular useful for speeding up iterative compilation
+when generating a source/build system tree.
+
+For instance:
+- first time, one generates a tree of files:
+  - generated: path/{t1,t2,t3}
+- then call tree_update_from_cache("path")
+  - will generate: __cache_path/{t1,t2,t3}
+  - and untouch: path/{t1,t2,t3}
+- second time, re-generate a tree of file:
+  - say generated files are identical: path/{t1,t2,t3}
+- then call tree_update_from_cache("path")
+  - will untouch in cache: __cache_path/{t1,t2,t3}
+  - and reset to previous timestamps files: path/{t1,t2,t3}
+- third time, re-generate again with some changes:
+  - say t1 is identical, t2 content has changed and no t3: path/{t1,t2'}
+- then call tree_update_from_cache("path")
+  - will update t2' and remove t3 in cache: __cache_path/{t1,t2'}
+  - and reset to previous timestamp t1: path/{t1,t2'}
+
+Note that by default the `dir`/__cache_`name` cache path is used
+for a given path `dir`/`name`.
+Though it is also possible to have the cache path inside the generated tree,
+in this case use for instance:
+
+    tree_update_from_cache(path, Path(path) / "__cache_src")
+
+For more evolved scenarii, specialize the provided FileTreeCache class.
+
+"""
+
+
+from pathlib import Path
+import shutil
+import sys
+import filecmp
+from typing import Optional, Union, List
+
+from .tree_utils import tree_move, tree_remove
+
+
+__all__ = [
+    "FileTreeCache",
+    "tree_update_from_cache",
+]
+
+def is_relative_to(path: Path, other: Path) -> bool:
+    """
+    Dynamically choose implementation based on Python version
+    """
+    # Python 3.9+
+    if sys.version_info >= (3, 9):
+        return path.is_relative_to(other)
+
+    # Python 3.8 and earlier
+    try:
+        path.relative_to(other)
+        return True
+    except ValueError:
+        return False
+
+
+class FileTreeCache():
+    """
+    Class for implementation of the file tree cache.
+    Can be derived to changes for instance default cache name/tmp name prefixes
+    or to specialize for other contexts.
+    """
+    default_cache_prefix = "__cache_"
+    default_tmp_cache_prefix = "__tmp_cache_"
+    default_tmp_prefix = "__tmp_"
+
+    def __init__(self,
+                 src_path: Union[str, Path],
+                 cache_path: Optional[Union[str, Path]] = None
+                 ) -> None:
+        self.src_path = Path(src_path).absolute()
+        self.cache_path = (
+            Path(cache_path).absolute()
+            if cache_path is not None else
+            (self.src_path.parent /
+             f"{self.default_cache_prefix}{self.src_path.name}")
+        )
+        ctx_msg = f"tree_cache: {src_path = }, {cache_path = }"
+        assert self.src_path != self.cache_path, f"src_path and cache_path must differ on {ctx_msg}"
+        assert not is_relative_to(self.src_path, self.cache_path), f"src_path must not be relative to cache_path on {ctx_msg}"
+        self._tmp_path = (
+            self.src_path.parent /
+            f"{self.default_tmp_prefix}{self.src_path.name}")
+        self._tmp_cache_path = (
+            self.src_path.parent /
+            f"{self.default_tmp_cache_prefix}{self.src_path.name}")
+
+    @classmethod
+    def _copytree_or_cache(cls, src_dir: Path, dst_dir: Path, cache_dir: Path, dst_cache_dir: Path) -> None:
+        assert not dst_dir.exists()
+        assert not dst_cache_dir.exists()
+        assert src_dir.is_dir()
+        assert not cache_dir.exists() or cache_dir.is_dir()
+        assert not is_relative_to(cache_dir, src_dir)
+
+        def copy_or_cache(src, dst):
+            base_src = Path(src).relative_to(src_dir)
+            cache_src = cache_dir / base_src
+            base_dst = Path(dst).relative_to(dst_dir)
+            cache_dst = dst_cache_dir / base_dst
+            cache_dst.parent.mkdir(parents=True, exist_ok=True)
+            if cache_src.exists() and filecmp.cmp(str(src), str(cache_src), shallow=False):
+                shutil.copy2(str(cache_src), str(cache_dst))
+                shutil.copy2(str(cache_src), dst)
+            else:
+                shutil.copy2(src, str(cache_dst))
+                shutil.copy2(src, dst)
+        shutil.copytree(str(src_dir), str(dst_dir), copy_function=copy_or_cache)
+
+    def update_from_cache(self) -> None:
+        assert self.src_path.exists(), f"src path must exist before swapping with cache"
+
+        # Move cache path apart first as it may be relative to source path
+        tree_move(self.cache_path, self._tmp_cache_path, ignore_missing=True, exist_ok=True)
+        # Move source path apart before recreating merged source tree
+        tree_move(self.src_path, self._tmp_path, exist_ok=True)
+
+        # Manage the source/cache merge to the dst/dst_cahe with a variant of
+        # copytree.
+        self._copytree_or_cache(
+            src_dir=self._tmp_path,
+            dst_dir=self.src_path,
+            cache_dir=self._tmp_cache_path,
+            dst_cache_dir=self.cache_path,
+        )
+
+        # Remove tmp source path
+        tree_remove(self._tmp_path)
+        # Note that the tmp cache path may not exist
+        tree_remove(self._tmp_cache_path, ignore_missing=True)
+
+
+def tree_update_from_cache(
+        src_path: Union[str, Path],
+        cache_path: Optional[Union[str, Path]] = None) -> None:
+    """
+    Update from cache the current generation of a tree from the
+    older generations, preserving file stamps when files contents are identical.
+
+    :param src_path: str or Path object to the generated tree
+    :param cache_path: optional str or Path object to the cache path,
+    or defaults to: `cache_path = src_path.parent / f"__cache_{src_path.name}"`
+    """
+    FileTreeCache(src_path, cache_path).update_from_cache()

aidge_core/testing/utils/tree_utils.py

+"""
+
+Provide utility function for file trees manipulations.
+
+"""
+
+import shutil
+import sys
+from pathlib import Path
+from typing import Union
+
+
+__all__ = [
+    "tree_move",
+    "tree_remove",
+]
+
+
+def is_relative_to(path: Path, other: Path) -> bool:
+    """
+    Dynamically choose implementation based on Python version
+    """
+    # Python 3.9+
+    if sys.version_info >= (3, 9):
+        return path.is_relative_to(other)
+
+    # Python 3.8 and earlier
+    try:
+        path.relative_to(other)
+        return True
+    except ValueError:
+        return False
+
+
+def tree_remove(
+        path: Union[str, Path],
+        ignore_missing: bool = False,
+) -> None:
+    """
+    Remove the full tree at path.
+    Optionally ignore if the path does not exist when ignore_missing is True.
+
+    :param path: str or Path object to the directory path
+    :param ignore_missing: if True will return early is path does not exists
+    """
+    path = Path(path)
+    ctx_msg = f"tree_remove: : {path = }"
+    assert ignore_missing or path.exists(), f"path must exists when ignore_missing is False on {ctx_msg}"
+    if ignore_missing and not path.exists():
+        return
+    shutil.rmtree(path)
+
+
+def tree_move(
+        src_path: Union[str, Path],
+        dst_path: Union[str, Path],
+        ignore_missing: bool = False,
+        exist_ok: bool = False,
+) -> None:
+    """
+    Move the whole src_path file tree to dst_path.
+    Optionally does nothing if the src path does not exists and ignore_missing is True.
+    Optionally the full dst_path will be removed first when exists_ok is True.
+
+    :param src_path: str or Path object to the source directory path
+    :param dst_path: str or Path object to the new path name for the source directory
+    :param ignore_missing: if True will return early is src_path does not exists
+    :param exist_ok: if True will first erase the new path name if it exists
+    """
+    src_path = Path(src_path)
+    dst_path = Path(dst_path)
+    ctx_msg = f"tree_move: : {src_path = }, {dst_path = }"
+    assert ignore_missing or src_path.exists(), f"src_path must exists when ignore_missing is False on {ctx_msg}"
+    assert exist_ok or not dst_path.exists(), f"dst_path must not exists when exist_ok is False on {ctx_msg}"
+    assert src_path != dst_path, f"paths must not be identical on {ctx_msg}"
+    assert not is_relative_to(dst_path, src_path), f"dst_path must not be relative to src_path on {ctx_msg}"
+    assert not is_relative_to(src_path, dst_path), f"src_path must not be relative to dst_path on {ctx_msg}"
+    if ignore_missing and not src_path.exists():
+        return
+    if exist_ok and dst_path.exists():
+        shutil.rmtree(dst_path)
+    shutil.move(src_path, dst_path)

aidge_core/unit_tests/static/main.cpp

+/*
+Example main.cpp used to test aidge export.
+This file is copied in the test export.
+*/
+#include <iostream>
+
+/* Register default cpu Tensor implementation */
+#include <aidge/backend/cpu/data/TensorImpl.hpp>
+
+/* Include model generator */
+#include "include/dnn.hpp"
+
+int main()
+{
+
+    std::cout << "BEGIN" << std::endl;
+
+    std::shared_ptr<Aidge::GraphView> graph = generateModel();
+
+    std::cout << "END" << std::endl;
+
+    return 0;
+}

aidge_core/unit_tests/test_parameters.py

@@ -27,23 +27,15 @@ class test_attributes(unittest.TestCase):
         out_channels = 8
         k_dims = [2, 2]
         conv_op = aidge_core.Conv2D(in_channels , out_channels, k_dims).get_operator()
-        self.assertEqual(conv_op.get_attr("InChannels"), in_channels)
-        self.assertEqual(conv_op.get_attr("OutChannels"), out_channels)
-        self.assertEqual(conv_op.get_attr("KernelDims"), k_dims)
+        self.assertEqual(conv_op.in_channels(), in_channels)
+        self.assertEqual(conv_op.out_channels(), out_channels)
+        self.assertEqual(conv_op.attr.get_attr("kernel_dims"), k_dims)
 
     def test_fc(self):
         in_channels = 4
         out_channels = 8
-        nb_bias = True
-        fc_op = aidge_core.FC(in_channels, out_channels, nb_bias).get_operator()
-        self.assertEqual(fc_op.get_attr("OutChannels"), out_channels)
-        self.assertEqual(fc_op.get_attr("NoBias"), nb_bias)
-
-    def test_matmul(self):
-        in_channels = 4
-        out_channels = 8
-        matmul_op = aidge_core.MatMul(in_channels, out_channels).get_operator()
-        self.assertEqual(matmul_op.get_attr("OutChannels"), out_channels)
+        fc_op = aidge_core.FC(in_channels, out_channels).get_operator()
+        self.assertEqual(fc_op.out_channels(), out_channels)
 
     def test_producer_1D(self):
         dims = [5]
@@ -73,7 +65,7 @@ class test_attributes(unittest.TestCase):
     def test_leaky_relu(self):
         negative_slope = 0.25
         leakyrelu_op = aidge_core.LeakyReLU(negative_slope).get_operator()
-        self.assertEqual(leakyrelu_op.get_attr("NegativeSlope"), negative_slope)
+        self.assertEqual(leakyrelu_op.attr.get_attr("negative_slope"), negative_slope)
 
 if __name__ == '__main__':
     unittest.main()