Merge pull request #296 from ICRAR/LIU-417

LIU-417: Add GraphConfig to translator

daliuge-translator/dlg/dropmake/dm_utils.py

@@ -168,8 +168,6 @@ def _check_MKN(m, k, n):
     if ratio1.is_integer() and ratio2.is_integer():
         return int(ratio1), int(ratio2)
     else:
-        from .pg_generator import GraphException
-
         raise GraphException("M-K and k-N must be pairs of multiples")
 
 

daliuge-translator/dlg/dropmake/graph_config.py

@@ -0,0 +1,118 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2015
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+
+"""
+Module containing utility methods for working with GraphConfigs
+"""
+
+import logging
+
+LOGGER = logging.getLogger(__name__)
+ACTIVE_CONFIG_KEY = "activeGraphConfigId"
+CONFIG_KEY = "graphConfigurations"
+GRAPH_NODES = "nodeDataArray"
+
+
+def apply_active_configuration(logical_graph: dict) -> dict:
+    """
+    Given a JSON representation of the LogicalGraph Template (LGT), apply the 
+    "Active Configuration" to the relevant constructs/fields.
+
+    :param: logical_graph, dict representation of JSON LGT
+
+    Currently, this does not raise any exceptions, but logs either warnings or errors. 
+    If there are missing keys or the configuration cannot be applied, it will return 
+    the original LGT. See graph_config.check_config_is_value for more details. 
+
+    return: dict, the updated LG
+    """
+    if is_config_invalid(logical_graph):
+        return logical_graph
+
+    try:
+        activeConfigurationID = logical_graph[ACTIVE_CONFIG_KEY]
+        activeConfig = logical_graph[CONFIG_KEY][activeConfigurationID]
+        nodeDataArray = logical_graph[GRAPH_NODES] 
+
+        for node_id, fields in activeConfig["nodes"].items():
+            idx = get_key_idx_from_list(node_id, nodeDataArray) 
+            if idx is None:
+                LOGGER.warning(
+                    "%s present in activeConfig but not available in Logical Graph.",
+                    node_id
+                )
+                continue 
+            node_name = nodeDataArray[idx]["name"]
+            for field_id, cfg_field in activeConfig["nodes"][node_id]["fields"].items():
+                fieldidx = get_key_idx_from_list(field_id, nodeDataArray[idx]["fields"])
+                field = nodeDataArray[idx]["fields"][fieldidx]
+                prev_value = field["value"]
+                field["value"] =  cfg_field["value"]
+                field_name = field["name"]
+                LOGGER.info("Updating: Node %s, Field %s, from %s to %s",
+                            node_name, field_name, str(prev_value), str(field["value"]))
+                nodeDataArray[idx]["fields"][fieldidx] = field
+        logical_graph[GRAPH_NODES] = nodeDataArray
+
+    except KeyError:
+        LOGGER.warning(
+            "Graph config key does not exist in logical graph. Using base field values."
+        )
+
+    return logical_graph
+
+def is_config_invalid(logical_graph: dict) -> bool:
+    """
+    Given a logical graph, verify that the correct keys are present prior to applying
+    the configuration. 
+
+    We want to make sure that: 
+        - "activeGraphConfigId" and "graphConfigurations" exist 
+        - Both of these store non-empty information
+
+    Note: We do not perform any validation on if the graphConfig is self-consistent; that 
+    is, if it mentions a graphConfig Key not in the LG, or there is a field/node ID that 
+    is not in the LG, we report this as an error and continue (this implies an error 
+    upstream that we cannot resolve).
+
+    :return: True if the config has correct keys and they are present. 
+    """
+
+    checkActiveId = logical_graph.get(ACTIVE_CONFIG_KEY)
+    if not checkActiveId:
+        LOGGER.warning("No %s data available in Logical Graph.", ACTIVE_CONFIG_KEY)
+    checkGraphConfig = logical_graph.get(CONFIG_KEY)    
+    if not checkGraphConfig:
+        LOGGER.warning("No %s data available in Logical Graph.", CONFIG_KEY)
+
+    return (not checkActiveId) or (not checkGraphConfig)
+
+
+def get_key_idx_from_list(key: str, dictList: list) -> int: 
+    """
+    Retrieve the index of the node that exists in the nodeDataArray sub-dict in the
+    Logical Graph. 
+
+    Note: This is necessary because we store each node in a list of dictionaries, rather 
+    than a dict of dicts. 
+    """
+    return next((idx for idx, keyDict in enumerate(dictList) if keyDict['id']==key), None)

daliuge-translator/dlg/dropmake/lg.py

@@ -53,8 +53,9 @@
     GInvalidNode,
     load_lg,
 )
-from .definition_classes import Categories
-from .lg_node import LGNode
+from dlg.dropmake.definition_classes import Categories
+from dlg.dropmake.lg_node import LGNode
+from dlg.dropmake.graph_config import apply_active_configuration
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +68,7 @@ class LG:
     it is doing all the conversion inside __init__
     """
 
-    def __init__(self, f, ssid=None):
+    def __init__(self, f, ssid=None, apply_config=True):
         """
         parse JSON into LG object graph first
         """
@@ -87,6 +88,9 @@ def __init__(self, f, ssid=None):
         logger.info("Loading graph: %s", lg["modelData"]["filePath"])
         logger.info("Found LG version: %s", lgver)
 
+        if apply_config:
+            lg = apply_active_configuration(lg)
+
         if LG_VER_EAGLE == lgver:
             lg = convert_mkn(lg)
             lg = convert_fields(lg)
@@ -101,6 +105,7 @@ def __init__(self, f, ssid=None):
             raise GraphException(
                 "Logical graph version '{0}' not supported!".format(lgver)
             )
+
         self._done_dict = {}
         self._group_q = collections.defaultdict(list)
         self._output_q = collections.defaultdict(list)

daliuge-translator/test/dropmake/test_graph_config.py

@@ -0,0 +1,132 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2015
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+
+import json
+import unittest
+from dlg.dropmake.graph_config import apply_active_configuration, get_key_idx_from_list
+import daliuge_tests.dropmake as test_graphs
+
+try:
+    from importlib.resources import files, as_file
+except ModuleNotFoundError:
+    from importlib_resources import files
+
+LOG_PRFIX = "WARNING:dlg.dropmake.graph_config:"
+
+def get_lg_from_fname(lg_name: str) -> dict:
+    """
+    Return the logical graph from the graph_config files in the test graph repository 
+    """
+    fname = str(files(test_graphs) / f"graph_config/{lg_name}")
+    with open(fname, "r") as fp:
+            return json.load(fp)
+
+def get_value_from_lgnode_field(node_id: str, field_id: str, logical_graph: dict) -> str:
+    """
+    Helper function that retrieves the value of a specified field from a given node
+
+    Returns: str representation of the value
+    """
+    idx = get_key_idx_from_list(node_id, logical_graph["nodeDataArray"])
+    field_idx = get_key_idx_from_list(field_id, logical_graph["nodeDataArray"][idx]["fields"])
+    return logical_graph["nodeDataArray"][idx]["fields"][field_idx]["value"]
+
+
+class TestGraphConfig(unittest.TestCase):
+    """
+    """
+
+    def test_apply_with_empty_config(self):
+        """
+        Exhaust the following possibilities that may exist when attempting to 
+        apply graph config:
+            - "activeGraphConfigId" is not in graph
+            - "activeGraphConfigId" is empty
+            - "graphConfigurations" is not in graph
+            - "graphConfigurations" is empty
+            - Keys from the activeGraphConfigId or graphConfigurations are not found in 
+            the logical graph. 
+        """ 
+        lg = get_lg_from_fname("ArrayLoopNoActiveID.graph")
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+        self.assertEqual(
+            [f"{LOG_PRFIX}No activeGraphConfigId data available in Logical Graph.",
+            f"{LOG_PRFIX}No graphConfigurations data available in Logical Graph."],
+            cm.output
+        )
+
+        lg['activeGraphConfigId'] = ""
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+        self.assertEqual(
+            [f"{LOG_PRFIX}No activeGraphConfigId data available in Logical Graph.",
+            f"{LOG_PRFIX}No graphConfigurations data available in Logical Graph."],
+            cm.output
+        )
+
+        lg['activeGraphConfigId'] = "temporaryString"
+
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+        self.assertEqual(
+            [f"{LOG_PRFIX}No graphConfigurations data available in Logical Graph."],
+            cm.output
+        )
+
+        lg['graphConfigurations'] = {"key": "value"}
+
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+            self.assertEqual(
+               [f"{LOG_PRFIX}Graph config key does not exist in logical graph. "
+                "Using base field values."],
+               cm.output
+            )
+
+    def test_apply_with_loop(self):
+        """
+        ArrayLoopLoop.graph has a GraphConfig with modified "num_of_iter" field in the 
+        "Loop" node (construct). 
+        """
+        lg = get_lg_from_fname("ArrayLoopLoop.graph")
+        node_id = "732df21b-f714-4d25-9773-b4169db270a0"
+        field_id = "3d25fcc9-50bb-4bbc-9b19-2eceabc238f2"
+        value = get_value_from_lgnode_field(node_id, field_id, lg)
+        self.assertEqual(1, int(value))
+        lg = apply_active_configuration(lg)
+        value = get_value_from_lgnode_field(node_id, field_id, lg)
+        self.assertEqual(5, int(value))
+
+    def test_apply_with_scatter(self):
+        """
+        ArrayLoopLoop.graph has a GraphConfig with modified "num_of_copies" field in the 
+        "Scatter" node (construct). 
+        """
+        lg = get_lg_from_fname("ArrayLoopScatter.graph")
+        node_id = "5560c56a-10f3-4d42-a436-404816dddf5f"
+        field_id = "0057b318-2405-4b5b-ac59-06c0068f91b7"
+        value = get_value_from_lgnode_field(node_id, field_id, lg)
+        self.assertEqual(1, int(value))
+        lg = apply_active_configuration(lg)
+        value = get_value_from_lgnode_field(node_id, field_id, lg)
+        self.assertEqual(5, int(value))