Merge branch 'merge-return-values' into 'ci-test'

Merge return values

See merge request !12

cimpy/cimexport.py

@@ -10,17 +10,19 @@ logger = logging.getLogger(__name__)
 
 
 # This function gets all attributes of an object and resolves references to other objects
-def _get_class_attributes_with_references(res, version, url_reference_dict):
+def _get_class_attributes_with_references(import_result, version):
     class_attributes_list = []
-
-    for key in res.keys():
-        class_dict = dict(name=res[key].__class__.__name__)
+    # extract topology and urls
+    topology = import_result['topology']
+    urls = import_result['meta_info']['urls']
+    for key in topology.keys():
+        class_dict = dict(name=topology[key].__class__.__name__)
         class_dict['mRID'] = key
         # array containing all attributes, attribute references to objects
-        attributes_dict = _get_attributes(res[key])
+        attributes_dict = _get_attributes(topology[key])
         # change attribute references to mRID of the object, res needed because classes like SvPowerFlow does not have
         # mRID as an attribute. Therefore the corresponding class has to be searched in the res dictionary
-        class_dict['attributes'] = _get_reference_uuid(attributes_dict, version, res, key, url_reference_dict)
+        class_dict['attributes'] = _get_reference_uuid(attributes_dict, version, topology, key, urls)
         class_attributes_list.append(class_dict)
         del class_dict
 
@@ -28,7 +30,7 @@ def _get_class_attributes_with_references(res, version, url_reference_dict):
 
 
 # This function resolves references to objects
-def _get_reference_uuid(attr_dict, version, res, mRID, url_reference_dict):
+def _get_reference_uuid(attr_dict, version, topology, mRID, urls):
     reference_list = []
     base_class_name = 'cimpy.' + version + '.Base'
     base_module = importlib.import_module(base_class_name)
@@ -48,7 +50,7 @@ def _get_reference_uuid(attr_dict, version, res, mRID, url_reference_dict):
                     # The % added before the mRID is used in the lambda _set_attribute_or_reference
                     if not hasattr(elem, 'mRID'):
                         # search for the object in the res dictionary and return the mRID
-                        UUID = '%' + _search_mRID(elem, res)
+                        UUID = '%' + _search_mRID(elem, topology)
                         if UUID == '%':
                             logger.warning('Object of type {} not found as reference for object with UUID {}.'.format(
                                 elem.__class__.__name__, mRID))
@@ -67,7 +69,7 @@ def _get_reference_uuid(attr_dict, version, res, mRID, url_reference_dict):
             if not hasattr(attr_dict[key], 'mRID'):
                 # search for object in res dict and return mRID
                 # The % added before the mRID is used in the lambda _set_attribute_or_reference
-                UUID = '%' + _search_mRID(attr_dict[key], res)
+                UUID = '%' + _search_mRID(attr_dict[key], topology)
                 if UUID == '%':
                     logger.warning('Object of type {} not found as reference for object with UUID {}.'.format(
                         elem.__class__.__name__, mRID))
@@ -77,11 +79,14 @@ def _get_reference_uuid(attr_dict, version, res, mRID, url_reference_dict):
         elif attr_dict[key] == "" or attr_dict[key] is None:
             pass
         else:
-            if mRID in url_reference_dict.keys():
-                if key.split('.')[1] in url_reference_dict[mRID]:
-                    attributes['value'] = '%URL%' + url_reference_dict[mRID][key.split('.')[1]][attr_dict[key]]
+            # attribute in urls dict?
+            if key.split('.')[1] in urls.keys():
+                # value in urls dict? should always be true
+                if attr_dict[key] in urls[key.split('.')[1]].keys():
+                    attributes['value'] = '%URL%' + urls[key.split('.')[1]][attr_dict[key]]
                 else:
-                    attributes['value'] = attr_dict[key]
+                    logger.warning('URL reference for attribute {} and value {} not found!'.format(
+                        key.split('.')[1], attr_dict[key]))
             else:
                 attributes['value'] = attr_dict[key]
 
@@ -101,8 +106,8 @@ def _get_reference_uuid(attr_dict, version, res, mRID, url_reference_dict):
 
 # This function searches a class_object in the res dictionary and returns the corresponding key (the mRID). Necessary
 # for classes without mRID as attribute like SvVoltage
-def _search_mRID(class_object, res):
-    for mRID, class_obj in res.items():
+def _search_mRID(class_object, topology):
+    for mRID, class_obj in topology.items():
         if class_object == class_obj:
             return mRID
     return ""
@@ -280,7 +285,7 @@ def _sort_classes_to_profile(class_attributes_list, activeProfileList):
     return export_dict, export_about_dict
 
 
-def cim_export(res, namespaces_dict, file_name, version, activeProfileList, url_reference_dict={}):
+def cim_export(import_result, file_name, version, activeProfileList):
     """Function for serialization of cgmes classes
 
     This function serializes cgmes classes with the template engine chevron. The classes are separated by their profile
@@ -288,15 +293,17 @@ def cim_export(res, namespaces_dict, file_name, version, activeProfileList, url_
     set_attributes_or_reference function is a lamda function for chevron to decide whether the value of an attribute is
     a reference to another class object or not.
 
-    :param res: a dictionary containing the cgmes classes accessible via the mRID
-    :param namespaces_dict: a dictionary containing the RDF namespaces used in the imported xml files
+    :param import_result: a dictionary containing the topology and meta information. The topology can be extracted via
+    import_result['topology']. The topology dictionary contains all objects accessible via their mRID. The meta
+    information can be extracted via import_result['meta_info']. The meta_info dictionary contains a new dictionary with
+    the keys: 'author', 'namespaces' and 'urls'. The last two are also dictionaries. 'urls' contains a mapping
+    between references to URLs and the extracted value of the URL, e.g. 'absoluteValue':
+    'http://iec.ch/TC57/2012/CIM-schema-cim16#OperationalLimitDirectionKind.absoluteValue' These mappings are accessible
+    via the name of the attribute, e.g. import_result['meta_info']['urls'}[attr_name] = {mapping like example above}.
+    'namespaces' is a dictionary containing all RDF namespaces used in the imported xml files.
     :param file_name: a string with the name of the xml files which will be created
     :param version: cgmes version, e.g. version = "cgmes_v2_4_15"
     :param activeProfileList: a list containing the strings of all short names of the profiles used for serialization
-    :param: url_reference_dict: a map containing a mapping between references to URLs and the extracted value of the
-        URL, e.g. 'absoluteValue': 'http://iec.ch/TC57/2012/CIM-schema-cim16#OperationalLimitDirectionKind.absoluteValue'
-        These mappings are accessible via the mRID of the class and the name of the attribute, e.g.
-        url_reference_dict[mRID][attribute_name] = {mapping like example above}
     """
 
     cwd = os.getcwd()
@@ -305,14 +312,14 @@ def cim_export(res, namespaces_dict, file_name, version, activeProfileList, url_
     logger.info('Start export procedure.')
 
     # returns all classes with their attributes and resolved references
-    class_attributes_list = _get_class_attributes_with_references(res, version, url_reference_dict)
+    class_attributes_list = _get_class_attributes_with_references(import_result, version)
 
     # determine class and attribute export profiles. The export dict contains all classes and their attributes where
     # the class definition and the attribute definitions are in the same profile. Every entry in about_dict generates
     # a rdf:about in another profile
     export_dict, about_dict = _sort_classes_to_profile(class_attributes_list, activeProfileList)
 
-    namespaces_list = _create_namespaces_list(namespaces_dict)
+    namespaces_list = _create_namespaces_list(import_result['meta_info']['namespaces'])
 
     # get information for Model header
     created = {'attr_name': 'created', 'value': datetime.now().strftime("%d/%m/%Y %H:%M:%S")}

cimpy/cimimport.py

@@ -21,12 +21,14 @@ def cim_import(xml_files, cgmes_version, start_dict=None):
     :param start_dict: a list of classes which indicates which classes will be read
         e.g. elements=["BaseVoltage", "ACLineSegment"]
         * If start_dict=None the complete file will be read
-    :return: res: map containing all classes contained in the xml file(s), assessable via the mRID
-    :return: namespaces: a map containing all RDF namespaces
-    :return: url_reference_dict: a map containing a mapping between references to URLs and the extracted value of the
-        URL, e.g. 'absoluteValue': 'http://iec.ch/TC57/2012/CIM-schema-cim16#OperationalLimitDirectionKind.absoluteValue'
-        These mappings are accessible via the mRID of the class and the name of the attribute, e.g.
-        url_reference_dict[mRID][attribute_name] = {mapping like example above}
+    :return: import_result: a dictionary containing the topology and meta information. The topology can be extracted via
+    import_result['topology']. The topology dictionary contains all objects accessible via their mRID. The meta
+    information can be extracted via import_result['meta_info']. The meta_info dictionary contains a new dictionary with
+    the keys: 'author', 'namespaces' and 'urls'. The last two are also dictionaries. 'urls' contains a mapping
+    between references to URLs and the extracted value of the URL, e.g. 'absoluteValue':
+    'http://iec.ch/TC57/2012/CIM-schema-cim16#OperationalLimitDirectionKind.absoluteValue' These mappings are accessible
+    via the name of the attribute, e.g. import_result['meta_info']['urls'}[attr_name] = {mapping like example above}.
+    'namespaces' is a dictionary containing all RDF namespaces used in the imported xml files.
     """
 
     # Import cim version class
@@ -35,35 +37,31 @@ def cim_import(xml_files, cgmes_version, start_dict=None):
     # Start the clock.
     t0 = time()
 
-    # map used to group errors
-    logger_errors_grouped = {}
+    # map used to group errors and infos
+    logger_grouped = dict(errors={}, info={})
 
-    # map used to group infos
-    logger_info_grouped = {}
+    # create a dict which will contain meta information and the topology
+    import_result = start_dict if start_dict is not None else dict(meta_info={}, topology={})
 
-    # A map of uuids to CIM objects to be returned.
-    res = start_dict if start_dict is not None else {}
-
-    # Obtain the namespaces from one of the input files
-    namespaces = _get_namespaces(xml_files[0])
-    namespace_rdf = _get_rdf_namespace(namespaces)
+    # create sub-dictionaries
+    import_result['meta_info'] = dict(namespaces=_get_namespaces(xml_files[0]), urls={})
+    namespace_rdf = _get_rdf_namespace(import_result['meta_info']['namespaces'])
 
     # CIM element tag base (e.g. {http://iec.ch/TC57/2012/CIM-schema-cim16#} )
-    base = "{" + namespaces["cim"] + "}"
+    base = "{" + import_result['meta_info']['namespaces']["cim"] + "}"
+
+    import_result, logger_grouped, = _instantiate_classes(import_result, xml_files, cgmes_version_path, namespace_rdf,
+                                                          base, logger_grouped)
 
-    res, logger_errors_grouped, logger_info_grouped = _instantiate_classes(res, xml_files, cgmes_version_path,
-                                                                           namespace_rdf, base, logger_errors_grouped,
-                                                                           logger_info_grouped)
-    res, url_reference_dict, logger_errors_grouped = _set_attributes(res, xml_files, namespace_rdf, base,
-                                                 logger_errors_grouped)
+    import_result, logger_grouped = _set_attributes(import_result, xml_files, namespace_rdf, base, logger_grouped)
 
-    if logger_errors_grouped:
-        for error, count in logger_errors_grouped.items():
+    if logger_grouped['errors']:
+        for error, count in logger_grouped['errors'].items():
             logging_message = '{} : {} times'.format(error, count)
             logger.warning(logging_message)
 
-    if logger_info_grouped:
-        for info, count in logger_info_grouped.items():
+    if logger_grouped['info']:
+        for info, count in logger_grouped['info'].items():
             logging_message = '{} : {} times'.format(info, count)
             logger.info(logging_message)
 
@@ -71,11 +69,11 @@ def cim_import(xml_files, cgmes_version, start_dict=None):
             print(logging_message)
 
     elapsed_time = time() - t0
-    logger.info('Created totally {} CIM objects in {}s\n\n'.format(len(res), elapsed_time))
+    logger.info('Created totally {} CIM objects in {}s\n\n'.format(len(import_result['topology']), elapsed_time))
     # print info of how many classes in total were instantiated to terminal
-    print('Created totally {} CIM objects in {}s'.format(len(res), elapsed_time))
+    print('Created totally {} CIM objects in {}s'.format(len(import_result['topology']), elapsed_time))
 
-    return res, namespaces, url_reference_dict
+    return import_result
 
 
 # This function instantiates the classes defined in all RDF files. All attributes are set to default values.
@@ -83,8 +81,12 @@ def cim_import(xml_files, cgmes_version, start_dict=None):
 # are set in the _set_attributes function because some attributes might be stored in one package and the class in
 # another. Since after this function all classes are instantiated, there should be no problem in setting the attributes.
 # Also the information from which package file a class was read is stored in the serializationProfile dictionary.
-def _instantiate_classes(res, xml_files, cgmes_version_path, namespace_rdf, base,
-                         logger_errors_grouped, logger_info_grouped):
+def _instantiate_classes(import_result, xml_files, cgmes_version_path, namespace_rdf, base,
+                         logger_grouped):
+
+    # extract topology from import_result
+    topology = import_result['topology']
+
     # length of element tag base
     m = len(base)
     # first step: create the dict res{uuid}=instance_of_the_cim_class
@@ -121,9 +123,9 @@ def _instantiate_classes(res, xml_files, cgmes_version_path, namespace_rdf, base
                     except ModuleNotFoundError:
                         error_msg = 'Module {} not implemented'.format(tag)
                         try:
-                            logger_errors_grouped[error_msg] += 1
+                            logger_grouped['errors'][error_msg] += 1
                         except KeyError:
-                            logger_errors_grouped[error_msg] = 1
+                            logger_grouped['errors'][error_msg] = 1
 
                         root.clear()
                         continue
@@ -132,50 +134,60 @@ def _instantiate_classes(res, xml_files, cgmes_version_path, namespace_rdf, base
                     klass = getattr(module, tag)
                     # Instantiate the class and map it to the uuid.
                     # res[uuid] = klass(UUID=uuid)
-                    res[uuid] = klass()
+                    topology[uuid] = klass()
                     info_msg = 'CIM object {} created'.format(module_name.split('.')[-1])
                     try:
-                        logger_info_grouped[info_msg] += 1
+                        logger_grouped['info'][info_msg] += 1
                     except KeyError:
-                        logger_info_grouped[info_msg] = 1
+                        logger_grouped['info'][info_msg] = 1
 
                     # check if the class has the attribute mRID and set the mRID to the read in UUID. If the class
                     # does not has this attribute, the UUID is only stored in the res dictionary.
-                    if hasattr(res[uuid], 'mRID'):
-                        res[uuid].mRID = uuid
+                    if hasattr(topology[uuid], 'mRID'):
+                        topology[uuid].mRID = uuid
 
                     if package is not '':
-                        res[uuid].serializationProfile['class'] = short_package_name[package]
+                        topology[uuid].serializationProfile['class'] = short_package_name[package]
                     else:
                         error_msg = 'Package information not found for class {}'.format(
                             klass.__class__.__name__
                         )
                         try:
-                            logger_errors_grouped[error_msg] += 1
+                            logger_grouped['errors'][error_msg] += 1
                         except KeyError:
-                            logger_errors_grouped[error_msg] = 1
+                            logger_grouped['errors'][error_msg] = 1
 
             # Check which package is read
-            elif event == "end" and 'Model.profile' in elem.tag:
-                for package_key in short_package_name.keys():
-                    if package_key in elem.text:
-                        package = package_key
-                        break
+            elif event == "end":
+                if 'Model.profile' in elem.tag:
+                    for package_key in short_package_name.keys():
+                        if package_key in elem.text:
+                            package = package_key
+                            break
+                # the author of all imported files should be the same, avoid multiple entries
+                elif 'author' in import_result['meta_info'].keys():
+                    pass
+                # extract author
+                elif 'Model.createdBy' in elem.tag:
+                    import_result['meta_info']['author'] = elem.text
+                elif 'Model.modelingAuthoritySet' in elem.tag:
+                    import_result['meta_info']['author'] = elem.text
 
             # Clear children of the root element to minimise memory usage.
             root.clear()
 
-    return res, logger_errors_grouped, logger_info_grouped
+    return import_result, logger_grouped
 
 
 # This function sets all attributes after the classes are instantiated by _instanciate_classes. Cyclic attributes like
 # PowerTransformerEnd <-> PowerTransformer are set. This function also stores the information from which package file
 # the attributes are read in the serializationProfile dictionary.
-def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
-    m = len(base)
+def _set_attributes(import_result, xml_files, namespace_rdf, base, logger_grouped):
 
-    # stores a mapping between references to URLs and the extracted attribute
-    url_reference_dict = {}
+    topology = import_result['topology']
+    urls = import_result['meta_info']['urls']
+
+    m = len(base)
 
     # Second step pass sets attributes and references.
     for xml_file in xml_files:
@@ -200,13 +212,13 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
                 if uuid is not None:
                     # Locate the CGMES object using the uuid.
                     try:
-                        obj = res[uuid]
+                        obj = topology[uuid]
                     except KeyError:
                         error_msg = 'Missing {} object with uuid: {}'.format(elem.tag[m:], uuid)
                         try:
-                            logger_errors_grouped[error_msg] += 1
+                            logger_grouped['errors'][error_msg] += 1
                         except KeyError:
-                            logger_errors_grouped[error_msg] = 1
+                            logger_grouped['errors'][error_msg] = 1
                         root.clear()
                         continue
 
@@ -223,9 +235,9 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
                                 if not hasattr(obj, attr):
                                     error_msg = "'%s' has not attribute '%s'" % (obj.__class__.__name__, attr)
                                     try:
-                                        logger_errors_grouped[error_msg] += 1
+                                        logger_grouped['errors'][error_msg] += 1
                                     except KeyError:
-                                        logger_errors_grouped[error_msg] = 1
+                                        logger_grouped['errors'][error_msg] = 1
                                     continue
 
                                 # Use the rdf:resource attribute to distinguish between attributes and references/enums.
@@ -256,14 +268,14 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
                                     # Use the '#' prefix to distinguish between references and enumerations.
                                     if uuid2[0] == "#":  # reference
                                         try:
-                                            val = res[uuid2[1:]]  # remove '#' prefix
+                                            val = topology[uuid2[1:]]  # remove '#' prefix
                                         except KeyError:
                                             error_msg = 'Referenced {} [{}] object missing.'.format(
                                                 obj.__class__.__name__, uuid2[1:])
                                             try:
-                                                logger_errors_grouped[error_msg] += 1
+                                                logger_grouped['errors'][error_msg] += 1
                                             except KeyError:
-                                                logger_errors_grouped[error_msg] = 1
+                                                logger_grouped['errors'][error_msg] = 1
 
                                             continue
 
@@ -286,9 +298,9 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
                                             error_msg = 'Multiplicity Error for class {} [{}], attribute {}. Multiplicity should be 1..1 or 0..1'.format(
                                                 obj.__class__.__name__, uuid, attr)
                                             try:
-                                                logger_errors_grouped[error_msg] += 1
+                                                logger_grouped['errors'][error_msg] += 1
                                             except KeyError:
-                                                logger_errors_grouped[error_msg] = 1
+                                                logger_grouped['errors'][error_msg] = 1
 
                                         if hasattr(val, obj.__class__.__name__):
                                             default1 = getattr(val, obj.__class__.__name__)
@@ -307,14 +319,19 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
                                                 error_msg = 'Multiplicity Error for class {} [{}], attribute {}. Multiplicity should be 1..1 or 0..1'.format(
                                                     val.__class__.__name__, uuid2[1:], obj.__class__.__name__)
                                                 try:
-                                                    logger_errors_grouped[error_msg] += 1
+                                                    logger_grouped['errors'][error_msg] += 1
                                                 except KeyError:
-                                                    logger_errors_grouped[error_msg] = 1
+                                                    logger_grouped['errors'][error_msg] = 1
 
                                     else:  # enum
                                         # if http in uuid2 reference to URL, create mapping
                                         if 'http' in uuid2:
-                                            url_reference_dict[uuid] = {attr: {uuid2.rsplit(".", 1)[1]: uuid2}}
+                                            if attr in urls.keys():
+                                                if uuid2.rsplit(".", 1)[1] not in urls[attr].keys():
+                                                    urls[attr][uuid2.rsplit(".", 1)[1]] = uuid2
+                                            else:
+                                                urls[attr] = {uuid2.rsplit(".", 1)[1]: uuid2}
+
                                             # url_reference_dict[uuid2.rsplit(".", 1)[1]] = uuid2
                                         val = uuid2.rsplit(".", 1)[1]
                                         setattr(obj, attr, val)
@@ -326,9 +343,9 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
                                         obj.__class__.__name__, attr
                                     )
                                     try:
-                                        logger_errors_grouped[error_msg] += 1
+                                        logger_grouped['errors'][error_msg] += 1
                                     except KeyError:
-                                        logger_errors_grouped[error_msg] = 1
+                                        logger_grouped['errors'][error_msg] = 1
                             else:  # if elem.get("{%s}ID" % nd_rdf is not None:
                                 # Finished setting object attributes.
                                 break
@@ -344,8 +361,7 @@ def _set_attributes(res, xml_files, namespace_rdf, base, logger_errors_grouped):
             root.clear()
 
         logger.info('END of parsing file "{}"'.format(xml_file))
-
-    return res, url_reference_dict, logger_errors_grouped
+    return import_result, logger_grouped
 
 
 # Returns a map of prefix to namespace for the given XML file.

examples/quickstart/exportCIGREMV.py

@@ -15,9 +15,9 @@ for file in xml_files:
 
 
 # res = cimpy.cimread(xml_files)
-res, namespaces, url_reference_dict = cimpy.cim_import(xml_files_abs, "cgmes_v2_4_15")
+import_result = cimpy.cim_import(xml_files_abs, "cgmes_v2_4_15")
 
 activeProfileList = ['DI', 'EQ', 'SV', 'TP']
 
 # dicts = cimpy.get_class_attributes_dict(res)
-cimpy.cim_export(res, namespaces, 'CIGREMV_reference_cgmes_v2_4_15', 'cgmes_v2_4_15', activeProfileList, url_reference_dict)
+cimpy.cim_export(import_result, 'CIGREMV_reference_cgmes_v2_4_15', 'cgmes_v2_4_15', activeProfileList)

examples/quickstart/importCIGREMV.py

@@ -15,9 +15,9 @@ for file in xml_files:
     xml_files_abs.append(os.path.abspath(file))
 
 # res = cimpy.cimread(xml_files)
-res, _, _ = cimpy.cim_import(xml_files_abs, "cgmes_v2_4_15")
+import_result = cimpy.cim_import(xml_files_abs, "cgmes_v2_4_15")
 print("\n\n")
 results = ["ACLineSegment", "PowerTransformer", "EnergyConsumer"]