added example python script

examples/call_python.sh

@@ -32,11 +32,11 @@
 source $HOME/hpc-scripts/prepareEnvironmentGcc.sh 
 
 ### Change Directory to your working directory (binaries, etc)
-cd $HOME
+cd $HOME/hpc-scripts/examples/
 
 start=`date`
 echo "Starting task $SLURM_ARRAY_TASK_ID at $start"
-python benchmarking.py $SLURM_ARRAY_TASK_ID ;rc=$?
+python call_python_script.py $SLURM_ARRAY_TASK_ID ;rc=$?
 end=`date`
 echo "Finished task $SLURM_ARRAY_TASK_ID with exit code $rc at $end"
 

examples/call_python_script.py

+import os, sys, subprocess, threading, time
+
+class CommandExecution(object):
+    """ Represents the execution of a single command line argument. """
+    def __init__(self):
+        self.timelimit = None
+        self.return_code = None
+        self.output = None
+        self.wall_time = None
+        self.proc = None
+
+    def stop(self):
+        self.timelimit = True
+        self.proc.kill()
+
+    def run(self, command_line_str, timelimit):
+        command_line_list = command_line_str.split()
+        command_line_list[0] = os.path.expanduser(command_line_list[0])
+        self.proc = subprocess.Popen(command_line_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        start_time = time.time()
+        timer = threading.Timer(timelimit, self.stop)
+        self.timelimit = False
+        self.output = ""
+        timer.start()
+        try:
+            stdout, stderr = self.proc.communicate()
+        except Exception as e:
+            self.output = self.output + "Error when executing the command:\n{}\n".format(e)
+        finally:
+            timer.cancel()
+            self.wall_time = time.time() - start_time
+            self.return_code = self.proc.returncode
+        self.output = self.output + stdout.decode('utf8')
+        if len(stderr) > 0:
+            self.output = self.output + "\n" + "#"*30 + "Output to stderr" + "#"*30 + "\n" + stderr.decode('utf8')
+        if self.timelimit and self.wall_time <= timelimit:
+            print("WARN: A timelimit was triggered although the measured time is {} seconds which is still below the time limit of {} seconds".format(self.wall_time, timelimit))
+
+
+def execute_command_line(command_line_str, timelimit):
+    """
+    Executes the given command line with the given time limit (in seconds).
+    :returns the output of the command (including the output to stderr, if present), the runtime of the command and either the return code or None (in case of a timelimit)
+    """
+    execution = CommandExecution()
+    execution.run(command_line_str, timelimit)
+    if execution.timelimit:
+        return execution.output, execution.wall_time, None
+    else:
+        return execution.output, execution.wall_time, execution.return_code
+
+
+def execute_and_log_command_line(command_line_str, timelimit, logfile):
+    """
+    Executes the given command line with the given time limit (in seconds) and stores the output the provided logfile.
+    :returns a string reporting the runtime, returncode and logfile location.
+    """
+    output, runtime, returncode = execute_command_line(command_line_str, timelimit)
+    header = "COMMAND: {}\nWALLTIME (seconds): {}\n".format(command_line_str, runtime)
+    if returncode == 0:
+        message = "done after {} seconds. Output saved to {}".format(runtime, logfile)
+        header += "\n"
+    elif returncode is None and runtime >= timelimit:
+        message = "timeout after {} seconds. Output saved to {}".format(runtime, logfile)
+        header += "TIMEOUT\n"
+    else:
+        message = "error after {} seconds with non-zero exit code {}. Output saved to {}".format(runtime,returncode,logfile)
+        header += "ERROR: {}\n".format(returncode)
+    output = header + "#"*30 + " LOG " + "#"*30 + "\n" + output
+    with open(logfile, 'w') as file:
+        file.write(output)      
+    return message
+
+timelimit = 3600
+logdir = "/rwthfs/rz/cluster/home/tq429871/logs/"
+
+# contains (logfilename, command)-pairs
+commands = []
+commands.append((r"eajs_P2-U3_pos-t8-c1-p0.01-e0.log", r"/home/tq429871/storm/build/bin/storm --prism /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/eajs/eajs02.prism --prop /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/eajs/eajs.props -const U=3 --multiobjective:schedrest positional -tm --exact --lpsolver gurobi --multiobjective:printres --gurobi:threads 8 --gurobi:concurrentmip 1 --multiobjective:precision 0.01 --multiobjective:maxsteps 0"))
+commands.append((r"resource-gathering_G10-E5-S100_pos-t1-c1-p0.01-e24.log", r"/home/tq429871/storm/build/bin/storm --prism /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/resource-gathering/resource-gathering.prism --prop /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/resource-gathering/resource-gathering.props -const REQGOLD=10,REQGEM=5,AVSTEPS=100 --multiobjective:schedrest positional -tm --exact --lpsolver gurobi --multiobjective:printres --gurobi:threads 1 --gurobi:concurrentmip 1 --multiobjective:precision 0.01 --multiobjective:maxsteps 24"))
+
+if __name__ == "__main__":
+    print("Found {} commands.".format(len(commands)))
+    if len(sys.argv) != 2:
+    	print("Invalid number of arguments. Expected 1 but got {}. Usage\npython {} <id>".format(len(sys.argv) - 1, sys.argv[0])) 
+    	exit(1)
+    try:
+    	id = int(sys.argv[1])
+    except ValueError:
+    	print("Unable to convert argument '{}' to an id".format(sys.argv[1]))
+    	exit(1)
+    if id < 0 or id >= len(commands):
+        print("id {} is out of range. Should be between 0 and {}".format(id, len(commands) - 1))
+        exit(1)
+    
+    sys.stdout.write("Command {}/{} ... ".format(id, len(commands) - 1))
+    sys.stdout.flush()
+    logfile = os.path.join(logdir, commands[id][0])
+    message = execute_and_log_command_line(commands[id][1], timelimit, logfile)
+    print(message)