Merge branch 'dev' into 'dev'

Add show_graphview funcionality.

See merge request !211

aidge_core/show_graphview.py

+import os
+import json
+import builtins
+import aidge_core
+import numpy as np
+from pathlib import Path
+ 
+def _retrieve_operator_attrs(node : aidge_core.Node) -> dict[str, int, float, bool, None]:
+    """
+    Returns the dictionary containing the attributes of a given Node.
+
+    :param graph: A Node in the list of ordered nodes. 
+    :type graph: aidge_core.Node
+
+    :return: A dictionary with the Node's attributes.
+    :rtype: dict[str, int, float, bool, None]
+    """       
+
+    if node.get_operator().attr is not None:
+        node_attr_dict =  node.get_operator().attr.dict()
+        for key,value in node_attr_dict.items():
+            if not type(value).__name__ in dir(builtins):
+                node_attr_dict[key] = value.name
+    
+    else:
+        node_attr_dict = {}
+
+    return node_attr_dict
+
+def _create_dict(ordered_nodes : list[aidge_core.Node], write_trainable_params_embed : bool, write_trainable_params_ext : bool, path_trainable_params : Path, params_file_format : str) -> dict[str, int, float, bool, None]:
+    """
+    Creates a dictionary to store the information of a given ordered GraphView. 
+
+    :param ordered_nodes: A list with the GraphView's ordered nodes.
+    :type graph: list
+    :param write_trainable_params_embed: Whether or not to write the eventual trainable parameters of the Nodes in the same file as the dict (embed). 
+    :type write_trainable_params_embed: bool
+    :param write_trainable_params_ext: Whether or not to write the eventual trainable parameters of the Nodes in an external file. 
+    :type write_trainable_params_ext: bool
+    :param path_trainable_params: Path of the external file used to store the Nodes' trainable parameters.
+    :type path_trainable_params: Path
+    :param params_file_format: Format of the external file used to store the Nodes' trainable parameters. Options: ``npz`` or ``json``. Default : ``json``. Requires ``write_trainable_params_ext``.
+    :type params_file_format: str
+    
+    :return: A dictionary with the GraphView description.
+    :rtype: dict[str, int, float, bool, None]
+    """            
+
+    graphview_dict = {'graph': []}
+
+    for node in ordered_nodes:
+        
+        if node is not None:
+            node_dict = {'name' : node.name(), 
+                         'optype' : node.get_operator().type(),
+                         'nb_inputs' : node.get_operator().nb_inputs(),
+                         'nb_outputs' : node.get_operator().nb_outputs()}
+            
+            inputs = []
+            for input_idx in range(node.get_operator().nb_inputs()):
+                input_dict = {'dims' : node.get_operator().get_input(input_idx).dims(),
+                              'data_type' : str(node.get_operator().get_input(input_idx).dtype()),
+                              'data_format' : str(node.get_operator().get_input(input_idx).dformat())}              
+                inputs.append(input_dict)    
+            
+            node_dict['inputs'] = inputs
+
+            outputs = []
+            for output_idx in range(node.get_operator().nb_outputs()):
+                output_dict = {'dims' : node.get_operator().get_output(output_idx).dims(),
+                               'data_type' : str(node.get_operator().get_output(output_idx).dtype()),
+                              'data_format' : str(node.get_operator().get_output(output_idx).dformat())}              
+                outputs.append(output_dict)    
+            
+            node_dict['outputs'] = outputs
+
+            parents = node.get_parents()
+            if None in parents:
+                if parents[0] is None: parents.append(parents.pop(0))
+            else:
+                pass
+    
+            parents_inputs = [] 
+            for parent in parents:
+                if parent is not None:
+                    for output_idx in range(parent.get_operator().nb_outputs()):
+                        for input_idx in range(node.get_operator().nb_inputs()):
+                            if parent.get_operator().get_output(output_idx).dims() == node.get_operator().get_input(input_idx).dims():
+                                parents_inputs.append((parent.name(), input_idx))
+
+                elif parent is None:
+                    for input_idx in list(range(node.get_operator().nb_inputs())):
+                        if input_idx not in [item[1] for item in parents_inputs]:
+                                parents_inputs.append((None, input_idx))  
+
+            parents_inputs.sort(key=lambda x: x[1])
+            node_dict['parents'] = parents_inputs
+
+            children_outputs = []
+            for child in node.get_children():
+                for input_idx in range(child.get_operator().nb_inputs()):
+                    for output_idx in range(node.get_operator().nb_outputs()):
+                        if child.get_operator().get_input(input_idx).dims() == node.get_operator().get_output(output_idx).dims():
+                            children_outputs.append((child.name(), output_idx))
+            node_dict['children'] = children_outputs
+        
+            # Check if my node is a metaop
+            attributes_dict = {}
+            if isinstance(node.get_operator(), aidge_core.MetaOperator_Op):
+                attributes_dict['micro_graph'] = []
+                for micro_node in node.get_operator().get_micro_graph().get_nodes():
+                    micro_node_dict = {'name' : micro_node.name(), 
+                                        'optype' : micro_node.type()}
+                    
+                    micro_node_attr_dict =  _retrieve_operator_attrs(micro_node)
+                    micro_node_dict['attributes'] = micro_node_attr_dict
+                    attributes_dict['micro_graph'].append(micro_node_dict)
+
+            else:
+                node_attr_dict = _retrieve_operator_attrs(node)
+                attributes_dict.update(node_attr_dict)
+
+            node_dict['attributes'] = attributes_dict
+
+            if node.type() == 'Producer':
+                if write_trainable_params_ext:
+                    
+                    params_file_format.casefold()
+
+                    if params_file_format=='npz':
+                        np.savez_compressed(Path(path_trainable_params, node.name()), **{node.name() : node.get_operator().get_output(0)})
+                        node_dict['tensor_data'] = Path(path_trainable_params, node.name() + '.npz')
+
+                    elif params_file_format=='json':
+                        tensor = np.array(node.get_operator().get_output(0))
+                        tensor_dict = {
+                            node.name() : 
+                            {
+                                'dims' : tensor.shape,
+                                'data_type' : str(tensor.dtype),
+                                'tensor_data' : tensor.tolist()
+                            }   
+                        }
+                                   
+                        with open(Path(path_trainable_params, node.name() + '.json'), 'w') as fp:
+                            json.dump(tensor_dict, fp, indent=4)
+
+                        node_dict['tensor_data'] = Path(path_trainable_params, node.name() + '.json')
+
+                    else:
+                        raise Exception("File format to write trainable parameters not recognized.")
+
+                
+                elif write_trainable_params_embed:
+                    node_dict['tensor_data'] = np.array(node.get_operator().get_output(0)).tolist()
+                
+                else:
+                    pass
+
+            graphview_dict['graph'].append(node_dict)
+
+        else: # node is None
+            pass
+    
+    return graphview_dict
+
+def _write_dict_json(graphview_dict : dict[str, int, float, bool, None], json_path : str) -> None:
+    """
+    Writes dictionary containing GraphView description to a JSON file.
+    
+    :param graphview_dict: A dictionary with the GraphView description.
+    :type graphview_dict: dict[str, int, float, bool, None]
+    :param json_path: Path to write JSON file.
+    :type json_path: str
+    """
+
+    with open(json_path, 'w') as fp:
+        json.dump(graphview_dict, fp, indent=4)
+
+    return None
+    
+def gview_to_json(gview : aidge_core.GraphView, json_path : Path, write_trainable_params_embed : bool = False, write_trainable_params_ext : bool = False, params_file_format : str = 'json') -> None:   
+    """
+    Generates the description for a GraphView in the JSON format.
+    
+    :param graph: A GraphView of Aidge.
+    :type graph: aidge_core.GraphView
+    :param json_path: Path to write JSON file.
+    :type json_path: Path
+    :param write_trainable_params_embed: Whether or not to write the eventual trainable parameters of the Nodes in the same file as the dict (embed). 
+    :type write_trainable_params_embed: bool, optional
+    :param write_trainable_params_ext: Whether or not to write the eventual trainable parameters of the Nodes in an external file. 
+    :type write_trainable_params_ext: bool, optional
+    :param params_file_format: Format of the external file used to store the Nodes' trainable parameters. Options: ``npz`` or ``json``. Default : ``json``. Requires ``write_trainable_params_ext``.
+    :type params_file_format: str, optional
+    """
+
+    if json_path.is_dir():
+        json_path = (json_path.parent).joinpath('model.json')
+
+    elif not json_path.is_dir():
+        if json_path.suffix == '.json':
+            pass
+        else: 
+            raise Exception('If ``json_path`` contains a filename it must be of JSON format.')
+
+    if write_trainable_params_ext:
+        path_trainable_params = (json_path.parent).joinpath(json_path.stem +  '_trainable_params/')
+    else:
+        path_trainable_params = Path()
+
+    if isinstance(gview, aidge_core.GraphView):
+        # Sort GraphView in topological order
+        ordered_nodes = gview.get_ordered_nodes()
+    
+        # Create dict from GraphView 
+        graphview_dict = _create_dict(ordered_nodes, write_trainable_params_embed, write_trainable_params_ext, path_trainable_params, params_file_format)
+        
+        # Write dict to JSON
+        _write_dict_json(graphview_dict, json_path)
+
+    else:
+        raise Exception("Graph must be an instance of aidge_core.GraphView.")
+        
+    return None
\ No newline at end of file

aidge_core/unit_tests/test_show_graphview.py

+import json
+import tempfile
+import unittest
+import builtins
+import aidge_core
+from pathlib import Path
+from aidge_core.show_graphview import gview_to_json
+
+def create_gview():
+    # Create a LeNet-like model
+    gview = aidge_core.sequential([aidge_core.PaddedConv2D(in_channels=1, out_channels=6, kernel_dims=[5,5], name='feature_feature_0_Conv', stride_dims=[1,1], padding_dims = [2,2,2,2]),
+                               aidge_core.ReLU(name='feature_feature_1_Relu'),
+                               aidge_core.MaxPooling2D(kernel_dims=[2,2], stride_dims=[2,2], ceil_mode=0, name='feature_feature_2_MaxPool'),
+                               aidge_core.Conv2D(in_channels=6, out_channels=16, kernel_dims=[5,5], name='feature_feature_3_Conv', stride_dims=[1,1], dilation_dims = [1,1]),
+                               aidge_core.ReLU(name='feature_feature_4_Relu'),
+                               aidge_core.MaxPooling2D(kernel_dims=[2,2], stride_dims=[2,2], ceil_mode=0, name='feature_feature_5_MaxPool'),
+                               aidge_core.FC(in_channels=400, out_channels=120, name='classifier_classifier_1_Gemm'),
+                               aidge_core.ReLU(name='classifier_classifier_2_Relu'),
+                               aidge_core.FC(in_channels=120, out_channels=84, name='classifier_classifier_3_Gemm'),
+                               aidge_core.ReLU(name='classifier_classifier_4_Relu'),
+                               aidge_core.FC(in_channels=84, out_channels=10, name='classifier_classifier_5_Gemm'),
+                            ])
+
+    # Fill Producers
+    for node in gview.get_nodes():
+        if node.type() == "Producer":
+            prod_op = node.get_operator()
+            value = prod_op.get_output(0)
+            value.set_backend("cpu")
+            tuple_out = node.output(0)[0]
+            
+            if (tuple_out[0].type() == "Conv" or tuple_out[0].type() == "PaddedConv") and tuple_out[1]==1:
+                # Conv weight
+                aidge_core.xavier_uniform_filler(value)
+            elif tuple_out[0].type() == "Conv" and tuple_out[1]==2:
+                # Conv bias
+                aidge_core.constant_filler(value, 0.01)
+            elif tuple_out[0].type() == "FC" and tuple_out[1]==1:
+                # FC weight
+                aidge_core.normal_filler(value)
+            elif tuple_out[0].type() == "FC" and tuple_out[1]==2:
+                # FC bias
+                aidge_core.constant_filler(value, 0.01)
+            else:
+                pass
+
+    # Compile model
+    gview.forward_dims([[1, 1, 28, 28]]) 
+    gview.set_datatype(aidge_core.dtype.float32)
+
+    return gview
+
+class test_show_gview(unittest.TestCase):
+    """Test aidge functionality to show GraphView.    
+    """
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_gview_to_json(self):
+        
+        gview = create_gview()
+
+        # Create temporary file to store JSON model description             
+        model_description_file = tempfile.NamedTemporaryFile(mode="w+", suffix='.json')
+
+        gview_to_json(gview, Path(model_description_file.name))
+
+        # Load JSON
+        with open(model_description_file.name, 'r') as fp:
+                model_json = json.load(fp)
+
+        # Get list of nodes of Aidge graphview
+        gview_ordered_nodes = gview.get_ordered_nodes()
+
+        # Iterate over the list of ordered nodes and the corresponding JSON 
+        self.assertEqual(len(gview_ordered_nodes), len(model_json['graph']))
+
+        for node_gview, node_json in zip(gview_ordered_nodes, model_json['graph']):   
+                    
+            self.assertEqual(node_gview.get_operator().type(), node_json['optype'])
+            self.assertEqual(node_gview.get_operator().nb_inputs(), node_json['nb_inputs'])
+            self.assertEqual(node_gview.get_operator().nb_outputs(), node_json['nb_outputs'])
+            
+            self.assertEqual(node_gview.get_operator().nb_inputs(), len(node_json['inputs']))
+            for input_idx in range(node_gview.get_operator().nb_inputs()):
+                self.assertEqual(node_gview.get_operator().get_input(input_idx).dims(), node_json['inputs'][input_idx]['dims'])
+                self.assertEqual(str(node_gview.get_operator().get_input(input_idx).dtype()), node_json['inputs'][input_idx]['data_type'])
+                self.assertEqual(str(node_gview.get_operator().get_input(input_idx).dformat()), node_json['inputs'][input_idx]['data_format'])
+
+            self.assertEqual(node_gview.get_operator().nb_outputs(), len(node_json['outputs']))
+            for output_idx in range(node_gview.get_operator().nb_outputs()):
+                self.assertEqual(node_gview.get_operator().get_output(output_idx).dims(), node_json['outputs'][output_idx]['dims'])
+                self.assertEqual(str(node_gview.get_operator().get_output(output_idx).dtype()), node_json['outputs'][output_idx]['data_type'])
+                self.assertEqual(str(node_gview.get_operator().get_output(output_idx).dformat()), node_json['outputs'][output_idx]['data_format'])
+
+            self.assertEqual(len(node_gview.get_parents()), len(node_json['parents']))                  
+            self.assertEqual(len(node_gview.get_children()), len(node_json['children']))
+
+            if not hasattr(node_gview.get_operator(), 'get_micro_graph'):
+                try:
+                    self.assertEqual(len(node_gview.get_operator().attr.dict()), len(node_json['attributes']))
+                    self.assertDictEqual(node_gview.get_operator().attr.dict(), node_json['attributes'])
+
+                except AttributeError:
+                    self.assertIsNone(node_gview.get_operator().attr) and self.assertFalse(node_json['attributes'])
+
+            elif hasattr(node_gview.get_operator(), 'get_micro_graph'):
+                
+                self.assertEqual(len(node_gview.get_operator().get_micro_graph().get_nodes()), len(node_json['attributes']['micro_graph']))
+                
+                for micro_node_gview in node_gview.get_operator().get_micro_graph().get_nodes():
+                    for micro_node_json in node_json['attributes']['micro_graph']:
+                        if micro_node_gview.get_operator().type() == micro_node_json['optype']:
+                            
+                            for key, value in micro_node_gview.get_operator().attr.dict().items():
+                                if not type(value).__name__ in dir(builtins):
+                                    # Replace original value by its name (str) because value is of a type that could not be written to the JSON
+                                    # Cannot update this dict inplace : micro_node_gview.get_operator().attr.dict().update({key : value.name}) 
+                                    temp_mnode_dict = micro_node_gview.get_operator().attr.dict()
+                                    temp_mnode_dict.update({key : value.name})
+                                    self.assertDictEqual(temp_mnode_dict, micro_node_json['attributes'])                
+                    
+if __name__ == '__main__':
+    unittest.main()
+

python_binding/data/pybind_Tensor.cpp

@@ -322,6 +322,7 @@ void init_Tensor(py::module& m){
     .def("grad", &Tensor::grad)
     .def("set_grad", &Tensor::setGrad)
     .def("dtype", &Tensor::dataType)
+    .def("dformat", &Tensor::dataFormat)
     .def("size", &Tensor::size)
     .def("capacity", &Tensor::capacity)
     .def("resize", (void (Tensor::*)(const std::vector<DimSize_t>&, std::vector<DimSize_t>)) &Tensor::resize, py::arg("dims"), py::arg("strides") = std::vector<DimSize_t>())

python_binding/graph/pybind_GraphView.cpp

@@ -146,6 +146,9 @@ void init_GraphView(py::module& m) {
           //                return py::none();
           //           }
           //      })
+          .def("get_ranked_nodes", &GraphView::getRankedNodes)
+          .def("set_dataformat", &GraphView::setDataFormat, py::arg("dataformat"))
+          
             ;
 
      m.def("get_connected_graph_view", &getConnectedGraphView);