cimexport.py 22.1 KB
Newer Older
1
2
3
4
import os
import importlib
import chevron
from datetime import datetime
5
from time import time
6
from cimpy.cgmes_v2_4_15.Base import Profile
7
import logging
8
import sys
Richard Marston's avatar
Richard Marston committed
9
10
from cimpy.cgmes_v2_4_15.Base import Base
cgmesProfile = Base.cgmesProfile
11

12
logger = logging.getLogger(__name__)
13

14

15
# This function gets all attributes of an object and resolves references to other objects
16
def _get_class_attributes_with_references(import_result, version):
17
    class_attributes_list = []
18
19
20
21
22
    # extract topology and urls
    topology = import_result['topology']
    urls = import_result['meta_info']['urls']
    for key in topology.keys():
        class_dict = dict(name=topology[key].__class__.__name__)
23
24
        class_dict['mRID'] = key
        # array containing all attributes, attribute references to objects
25
        attributes_dict = _get_attributes(topology[key])
26
27
        # change attribute references to mRID of the object, res needed because classes like SvPowerFlow does not have
        # mRID as an attribute. Therefore the corresponding class has to be searched in the res dictionary
28
        class_dict['attributes'] = _get_reference_uuid(attributes_dict, version, topology, key, urls)
29
30
31
32
        class_attributes_list.append(class_dict)
        del class_dict

    return class_attributes_list
33
34


35
# This function resolves references to objects
36
def _get_reference_uuid(attr_dict, version, topology, mRID, urls):
37
38
39
40
41
    reference_list = []
    base_class_name = 'cimpy.' + version + '.Base'
    base_module = importlib.import_module(base_class_name)
    base_class = getattr(base_module, 'Base')
    for key in attr_dict:
42
        if key in ['serializationProfile', 'possibleProfileList']:
43
44
45
            reference_list.append({key: attr_dict[key]})
            continue

46
        attributes = {}
47
        if isinstance(attr_dict[key], list):  # many
48
49
50
            array = []
            for elem in attr_dict[key]:
                if issubclass(type(elem), base_class):
51
52
53
                    # classes like SvVoltage does not have an attribute called mRID, the mRID is only stored as a key
                    # for this object in the res dictionary
                    # The % added before the mRID is used in the lambda _set_attribute_or_reference
54
                    if not hasattr(elem, 'mRID'):
55
                        # search for the object in the res dictionary and return the mRID
56
                        UUID = '%' + _search_mRID(elem, topology)
57
                        if UUID == '%':
Philipp Reuber's avatar
Philipp Reuber committed
58
                            logger.warning('Object of type {} not found as reference for object with UUID {}.'.format(
59
                                elem.__class__.__name__, mRID))
60
61
62
                    else:
                        UUID = '%' + elem.mRID

63
64
                    array.append(UUID)
                else:
Philipp Reuber's avatar
Philipp Reuber committed
65
                    logger.warning('Reference object not subclass of Base class for object with UUID {}.'.format(mRID))
66
67
68
69
            if len(array) == 1:
                attributes['value'] = array[0]
            else:
                attributes['value'] = array
70
        elif issubclass(type(attr_dict[key]), base_class):  # 0..1, 1..1
71
            # resource = key + ' rdf:resource='
72
            if not hasattr(attr_dict[key], 'mRID'):
73
74
                # search for object in res dict and return mRID
                # The % added before the mRID is used in the lambda _set_attribute_or_reference
75
                UUID = '%' + _search_mRID(attr_dict[key], topology)
76
                if UUID == '%':
Philipp Reuber's avatar
Philipp Reuber committed
77
                    logger.warning('Object of type {} not found as reference for object with UUID {}.'.format(
78
                        attr_dict[key].__class__.__name__, mRID))
79
80
            else:
                UUID = '%' + attr_dict[key].mRID
81
82
83
84
            attributes['value'] = UUID
        elif attr_dict[key] == "" or attr_dict[key] is None:
            pass
        else:
85
86
87
88
89
            # attribute in urls dict?
            if key.split('.')[1] in urls.keys():
                # value in urls dict? should always be true
                if attr_dict[key] in urls[key.split('.')[1]].keys():
                    attributes['value'] = '%URL%' + urls[key.split('.')[1]][attr_dict[key]]
90
                else:
91
92
                    logger.warning('URL reference for attribute {} and value {} not found!'.format(
                        key.split('.')[1], attr_dict[key]))
93
94
            else:
                attributes['value'] = attr_dict[key]
95
96
97
98
99

        attributes['attr_name'] = key
        if 'value' in attributes.keys():
            if isinstance(attributes['value'], list):
                for reference_item in attributes['value']:
100
101
                    # ignore default values
                    if reference_item not in ['', None, 0.0, 0]:
102
                        reference_list.append({'value': reference_item, 'attr_name': key})
103
            # ignore default values
104
            elif attributes['value'] not in ['', None, 0.0, 0, 'list']:
105
106
                reference_list.append(attributes)

107
    return reference_list
108
109


110
111
# This function searches a class_object in the res dictionary and returns the corresponding key (the mRID). Necessary
# for classes without mRID as attribute like SvVoltage
112
113
def _search_mRID(class_object, topology):
    for mRID, class_obj in topology.items():
114
115
116
117
118
        if class_object == class_obj:
            return mRID
    return ""


119
# Lambda function for chevron renderer to decide whether the current element is a reference or an attribute
120
121
122
123
124
def _set_attribute_or_reference(text, render):
    result = render(text)
    result = result.split('@')
    value = result[0]
    attr_name = result[1]
125
126
127
128
    if '%URL%' in value:
        reference = value.split('%')[2]
        return ' rdf:resource="' + reference + '"/>'
    elif '%' in value:
129
130
131
132
133
134
        reference = value.split('%')[1]
        return ' rdf:resource="#' + reference + '"/>'
    else:
        return '>' + value + '</cim:' + attr_name + '>'


135
# Lambda function for chevron renderer to set an attribute or a reference in the model description.
136
137
138
139
140
141
142
143
144
145
146
147
def _set_attribute_or_reference_model(text, render):
    result = render(text)
    result = result.split('@')
    value = result[0]
    attr_name = result[1]
    if '%' in value:
        reference = value.split('%')[1]
        return ' rdf:resource="' + reference + '"/>'
    else:
        return '>' + value + '</md:Model.' + attr_name + '>'


148
# Restructures the namespaces dict into a list. The template engine writes each entry in the RDF header
149
150
151
152
def _create_namespaces_list(namespaces_dict):
    namespaces_list = []

    for key in namespaces_dict:
153
        namespace = dict(key=key, url=namespaces_dict[key])
154
155
156
157
158
        namespaces_list.append(namespace)

    return namespaces_list


159
# This function sorts the classes and their attributes to the corresponding profiles. Either the classes/attributes are
160
161
# imported or they are set afterwards. In the first case the serializationProfile is used to determine from which
# profile this class/attribute was read. If an entry exists the class/attribute is added to this profile. In the
162
163
164
# possibleProfileList dictionary the possible origins of the class/attributes is stored. All profiles have a different
# priority which is stored in the enum cgmesProfile. As default the smallest entry in the dictionary is used to
# determine the profile for the class/attributes.
165
def _sort_classes_to_profile(class_attributes_list, activeProfileList):
166
167
168
169
170
171
172
173
    export_dict = {}
    export_about_dict = {}

    # iterate over classes
    for klass in class_attributes_list:
        same_package_list = []
        about_dict = {}

174
175
176
        # store serializationProfile and possibleProfileList
        # serializationProfile class attribute, same for multiple instances of same class, only last origin of variable stored
        serializationProfile = klass['attributes'][0]['serializationProfile']
177
178
        possibleProfileList = klass['attributes'][1]['possibleProfileList']

179
180
        class_serializationProfile = ''

181
        if 'class' in serializationProfile.keys():
182
            # class was imported
183
            if Profile[serializationProfile['class']] in activeProfileList:
184
                # else: class origin profile not active for export, get active profile from possibleProfileList
185
                if Profile[serializationProfile['class']].value in possibleProfileList[klass['name']]['class']:
186
187
188
                    # profile active and in possibleProfileList
                    # else: class should not have been imported from this profile, get allowed profile
                    # from possibleProfileList
189
                    class_serializationProfile = serializationProfile['class']
Philipp Reuber's avatar
Philipp Reuber committed
190
191
                else:
                    logger.warning('Class {} was read from profile {} but this profile is not possible for this class'
192
                                   .format(klass['name'], serializationProfile['class']))
Philipp Reuber's avatar
Philipp Reuber committed
193
194
            else:
                logger.info('Class {} was read from profile {} but this profile is not active for the export. Use'
195
                            'default profile from possibleProfileList.'.format(klass['name'], serializationProfile['class']))
196
197

        if class_serializationProfile == '':
198
            # class was created
199
200
            if klass['name'] in possibleProfileList.keys():
                if 'class' in possibleProfileList[klass['name']].keys():
201
                    possibleProfileList[klass['name']]['class'].sort()
202
                    for klass_profile in possibleProfileList[klass['name']]['class']:
203
                        if Profile(klass_profile).name in activeProfileList:
204
                            # active profile for class export found
205
                            class_serializationProfile = Profile(klass_profile).name
206
207
208
209
210
211
                            break
                    if class_serializationProfile == '':
                        # no profile in possibleProfileList active
                        logger.warning('All possible export profiles for class {} not active. Skip class for export.'
                                       .format(klass['name']))
                        continue
212
                else:
213
                    logger.warning('Class {} has no profile to export to.'.format(klass['name']))
214
            else:
215
                logger.warning('Class {} has no profile to export to.'.format(klass['name']))
216
217
218
219
220
221

        # iterate over attributes
        for attribute in klass['attributes']:
            if 'attr_name' in attribute.keys():
                attribute_class = attribute['attr_name'].split('.')[0]
                attribute_name = attribute['attr_name'].split('.')[1]
222
223
224
225
226

                # IdentifiedObject.mRID is not exported as an attribute
                if attribute_name == 'mRID':
                    continue

227
                attribute_serializationProfile = ''
228

229
                if attribute_name in serializationProfile.keys():
230
                    # attribute was imported
231
232
                    if Profile[serializationProfile[attribute_name]] in activeProfileList:
                        attr_value = Profile[serializationProfile[attribute_name]].value
233
234
                        if attr_value in possibleProfileList[attribute_class][attribute_name]:
                            attribute_serializationProfile = serializationProfile[attribute_name]
235

236
                if attribute_serializationProfile == '':
237
                    # attribute was added
238
239
                    if attribute_class in possibleProfileList.keys():
                        if attribute_name in possibleProfileList[attribute_class].keys():
240
                            possibleProfileList[attribute_class][attribute_name].sort()
241
                            for attr_profile in possibleProfileList[attribute_class][attribute_name]:
242
                                if Profile(attr_profile) in activeProfileList:
243
                                    # active profile for class export found
244
                                    attribute_serializationProfile = Profile(attr_profile).name
245
246
247
248
249
250
251
                                    break
                            if attribute_serializationProfile == '':
                                # no profile in possibleProfileList active, skip attribute
                                logger.warning('All possible export profiles for attribute {}.{} of class {} '
                                               'not active. Skip attribute for export.'
                                               .format(attribute_class, attribute_name, klass['name']))
                                continue
252
                        else:
253
254
                            logger.warning('Attribute {}.{} of class {} has no profile to export to.'.
                                           format(attribute_class, attribute_name, klass['name']))
255
                    else:
256
                        logger.warning('The class {} for attribute {} is not in the possibleProfileList'.format(
257
                            attribute_class, attribute_name))
258

259
                if attribute_serializationProfile == class_serializationProfile:
260
                    # class and current attribute belong to same profile
261
262
                    same_package_list.append(attribute)
                else:
263
264
                    # class and current attribute does not belong to same profile -> rdf:about in
                    # attribute origin profile
265
266
                    if attribute_serializationProfile in about_dict.keys():
                        about_dict[attribute_serializationProfile].append(attribute)
267
                    else:
268
                        about_dict[attribute_serializationProfile] = [attribute]
269

270
        # add class with all attributes in the same profile to the export dict sorted by the profile
271
        if class_serializationProfile in export_dict.keys():
272
            export_class = dict(name=klass['name'], mRID=klass['mRID'], attributes=same_package_list)
273
            export_dict[class_serializationProfile]['classes'].append(export_class)
274
275
276
            del export_class
        else:
            export_class = dict(name=klass['name'], mRID=klass['mRID'], attributes=same_package_list)
277
            export_dict[class_serializationProfile] = {'classes': [export_class]}
278

279
        # add class with all attributes defined in another profile to the about_key sorted by the profile
280
281
282
283
284
285
286
287
288
289
290
        for about_key in about_dict.keys():
            if about_key in export_about_dict.keys():
                export_about_class = dict(name=klass['name'], mRID=klass['mRID'], attributes=about_dict[about_key])
                export_about_dict[about_key]['classes'].append(export_about_class)
            else:
                export_about_class = dict(name=klass['name'], mRID=klass['mRID'], attributes=about_dict[about_key])
                export_about_dict[about_key] = {'classes': [export_about_class]}

    return export_dict, export_about_dict


291
def cim_export(import_result, file_name, version, activeProfileList):
292
293
    """Function for serialization of cgmes classes

294
295
    This function serializes cgmes classes with the template engine chevron. The classes are separated by their profile
    and one xml file for each profile is created. The package name is added after the file name. The
296
297
298
    set_attributes_or_reference function is a lamda function for chevron to decide whether the value of an attribute is
    a reference to another class object or not.

Jonathan Klimt's avatar
Jonathan Klimt committed
299
300
301
302
303
304
305
    :param import_result: a dictionary containing the topology and meta information. The topology can be extracted via \
    import_result['topology']. The topology dictionary contains all objects accessible via their mRID. The meta \
    information can be extracted via import_result['meta_info']. The meta_info dictionary contains a new dictionary with \
    the keys: 'author', 'namespaces' and 'urls'. The last two are also dictionaries. 'urls' contains a mapping \
    between references to URLs and the extracted value of the URL, e.g. 'absoluteValue': \
    'http://iec.ch/TC57/2012/CIM-schema-cim16#OperationalLimitDirectionKind.absoluteValue' These mappings are accessible \
    via the name of the attribute, e.g. import_result['meta_info']['urls'}[attr_name] = {mapping like example above}. \
306
    'namespaces' is a dictionary containing all RDF namespaces used in the imported xml files.
307
    :param file_name: a string with the name of the xml files which will be created
308
    :param version: cgmes version, e.g. version = "cgmes_v2_4_15"
309
    :param activeProfileList: a list containing the strings of all short names of the profiles used for serialization
310
    """
311

312
313
    cwd = os.getcwd()
    os.chdir(os.path.dirname(__file__))
314
315
    t0 = time()
    logger.info('Start export procedure.')
316

317
318
    profile_list = list(map(lambda a: Profile[a], activeProfileList))

319
    # iterate over all profiles
320
    for profile in profile_list:
321

322
        # File name
323
        full_file_name = file_name + '_' + profile.long_name() + '.xml'
324
325
326
327

        full_path = os.path.join(cwd, full_file_name)

        if not os.path.exists(full_path):
328
329
            output = generate_xml(import_result, version, file_name, profile, profile_list)

330
            with open(full_path, 'w') as file:
331
332
                logger.info('Write file \"%s\"', full_path)

333
                file.write(output)
334
        else:
335
            logger.error('File {} already exists in path {}. Delete file or change file name to serialize CGMES '
336
                           'classes.'.format(full_file_name, cwd))
337
338
339
340
341
            print('[ERROR:] File {} already exists in path {}. Delete file or change file name to serialize CGMES '
                           'classes.'.format(full_file_name, cwd), file=sys.stderr)
            os.chdir(cwd)
            exit(-1)

342
    os.chdir(cwd)
343
    logger.info('End export procedure. Elapsed time: {}'.format(time() - t0))
344
345


346
def generate_xml(cim_data, version, model_name, profile, available_profiles):
347
348
349
350
351
352
    """Function for serialization of cgmes classes

    This function serializes cgmes classes with the template engine chevron and returns them as a string.

    :param cim_data: a dictionary containing the topology and meta information. It can be created via :func:`~cimimport.cimimport()`
    :param version: cgmes version, e.g. version = "cgmes_v2_4_15"
353
    :param profile: The :class:`~cgmes.Profile` for which the serialization should be generated. . Possible values are TODO: enum
354
    :param model_name: a string with the name of the model.
355
    :param available_profiles: a list of all :class:`~cgmes.Profile`s in `cim_data`
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
    """

    # returns all classes with their attributes and resolved references
    class_attributes_list = _get_class_attributes_with_references(
        cim_data, version)

    # determine class and attribute export profiles. The export dict contains all classes and their attributes where
    # the class definition and the attribute definitions are in the same profile. Every entry in about_dict generates
    # a rdf:about in another profile
    export_dict, about_dict = _sort_classes_to_profile(
        class_attributes_list, available_profiles)

    namespaces_list = _create_namespaces_list(
        cim_data['meta_info']['namespaces'])

371
    if profile.name not in export_dict.keys() and profile.name not in about_dict.keys():
372
        raise RuntimeError("Profile " + profile.name + " not available for export, export_dict=" + str(export_dict.keys()) + ' and about_dict='+ str(about_dict.keys()) + '.')
373
374

    # extract class lists from export_dict and about_dict
375
376
    if profile.name in export_dict.keys():
        classes = export_dict[profile.name]['classes']
377
378
379
    else:
        classes = False

380
381
    if profile.name in about_dict.keys():
        about = about_dict[profile.name]['classes']
382
383
384
385
386
387
388
389
390
391
392
393
    else:
        about = False

    #Model header
    model_description = {
        'mRID': model_name,
        'description': [
            {'attr_name': 'created', 'value': datetime.now().strftime(
                "%d/%m/%Y %H:%M:%S")},
            {'attr_name': 'modelingAuthoritySet',
             'value': 'www.acs.eonerc.rwth-aachen.de'},
            {'attr_name': 'profile',
394
             'value': profile.long_name()}
395
396
397
398
399
400
401
402
403
404
405
406
407
        ]
    }

    with open('export_template.mustache') as f:
        output = chevron.render(f, {"classes": classes,
                                    "about": about,
                                    "set_attributes_or_reference": _set_attribute_or_reference,
                                    "set_attributes_or_reference_model": _set_attribute_or_reference_model,
                                    "namespaces": namespaces_list,
                                    "model": [model_description]})
    del model_description
    return output

408
# This function extracts all attributes from class_object in the form of Class_Name.Attribute_Name
409
410
411
412
def _get_attributes(class_object):
    inheritance_list = [class_object]
    class_type = type(class_object)
    parent = class_object
413
414

    # get parent classes
415
416
417
418
419
420
421
    while 'Base.Base' not in str(class_type):
        parent = parent.__class__.__bases__[0]()
        # insert parent class at beginning of list, classes inherit from top to bottom
        inheritance_list.insert(0, parent)
        class_type = type(parent)

    # dictionary containing all attributes with key: 'Class_Name.Attribute_Name'
422
    attributes_dict = dict(serializationProfile=class_object.serializationProfile, possibleProfileList={})
423

424
425
426
    # __dict__ of a subclass returns also the attributes of the parent classes
    # to avoid multiple attributes create list with all attributes already processed
    attributes_list = []
427

428
    # iterate over parent classes from top to bottom
429
430
431
432
    for parent_class in inheritance_list:
        # get all attributes of the current parent class
        parent_attributes_dict = parent_class.__dict__
        class_name = parent_class.__class__.__name__
433
434

        # check if new attribute or old attribute
435
436
437
438
439
440
441
442
        for key in parent_attributes_dict.keys():
            if key not in attributes_list:
                attributes_list.append(key)
                attributes_name = class_name + '.' + key
                attributes_dict[attributes_name] = getattr(class_object, key)
            else:
                continue

443
        # get all possibleProfileLists from all parent classes except the Base class (no attributes)
444
445
        # the serializationProfile from parent classes is not needed because entries in the serializationProfile
        # are only generated for the inherited class
446
        if class_name != 'Base':
447
448
            attributes_dict['possibleProfileList'][class_name] = parent_class.possibleProfileList

449
    return attributes_dict