Aufgrund einer Wartung wird GitLab am 18.01. zwischen 8:00 und 9:00 Uhr kurzzeitig nicht zur Verfügung stehen. / Due to maintenance, GitLab will be temporarily unavailable on 18.01. between 8:00 and 9:00 am.

jobfile.py 2.51 KB
Newer Older
1
import json
2
import os
3
import subprocess
Lukas Weber's avatar
Lukas Weber committed
4
import errno
5
from collections import defaultdict
6

Lukas Weber's avatar
Lukas Weber committed
7

8
'''Helpers for handling loadleveller jobfiles/scripts. For lack of a better idea, the job description files of loadleveller are actually executables that output a more verbose json parameter file to stdout. Use the taskmaker module to write the input scripts.'''
9
10
11

class JobFileGenError(Exception):
    pass
12

13
14
15
16
class JobFileOverwriteError(Exception):
    def __init__(self, difference):
        self.difference = difference

17
18
class JobFile:
    def __init__(self, filename):
19
20
21
        env = dict(os.environ)
        env['PATH'] += ':.'
        result = subprocess.run([filename], stdout=subprocess.PIPE, env=env)
22
23
24
25
26
27

        self.raw_jobfile = result.stdout.decode('utf-8')
        if result.returncode != 0:
            raise JobFileGenError('Generation script "{}" had a non-zero return code. Treating as error.'.format(filename))
 
        try:
28
            parsed_job = json.loads(self.raw_jobfile)
29
30
31
32
            self.__dict__.update(parsed_job)
        except Exception as e: 
            raise JobFileGenError('Could not parse job generation script output: {}'.format(e))

33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
    def _compare_to_old(self, old_data):
        diff = defaultdict(dict)
        for taskname, task in self.tasks.items():
            if taskname in old_data['tasks']:
                old_task = old_data['tasks'][taskname]
                for param, value in task.items():
                    if param not in old_task:
                        diff[taskname][param] = (value, None)
                    elif value != old_task[param]:
                        diff[taskname][param] = (value, old_task[param])
        return diff

    def write_job_input_file(self, force_overwrite):
        datadir = self.jobname + '.data'
        try:
            os.makedirs(datadir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        job_input_filename = os.path.join(datadir, 'parameters.json')

        if os.path.isfile(job_input_filename):
            with open(job_input_filename, 'r') as f:
                old_data = json.load(f)
            diff = self._compare_to_old(old_data)
            if not force_overwrite and len(diff) > 0:
                raise JobFileOverwriteError(difference=diff)
                 
61
62
63
        try:
            with open(job_input_filename, 'w') as f:
                f.write(self.raw_jobfile)
64
        except JobFileGenError as e:
65
            raise JobFileGenError('Could not write parameters.json: {}'.format(e))
66
67

        return job_input_filename