From 98541b51aaa984cd1ac177ce9922d1f8104cbcf1 Mon Sep 17 00:00:00 2001
From: Tim Quatmann <tim.quatmann@cs.rwth-aachen.de>
Date: Tue, 16 Jul 2019 11:53:34 +0200
Subject: [PATCH] added example python script
---
examples/call_python.sh | 4 +-
examples/call_python_script.py | 100 +++++++++++++++++++++++++++++++++
2 files changed, 102 insertions(+), 2 deletions(-)
create mode 100644 examples/call_python_script.py
diff --git a/examples/call_python.sh b/examples/call_python.sh
index 213ac58..279fa02 100755
--- a/examples/call_python.sh
+++ b/examples/call_python.sh
@@ -32,11 +32,11 @@
source $HOME/hpc-scripts/prepareEnvironmentGcc.sh
### Change Directory to your working directory (binaries, etc)
-cd $HOME
+cd $HOME/hpc-scripts/examples/
start=`date`
echo "Starting task $SLURM_ARRAY_TASK_ID at $start"
-python benchmarking.py $SLURM_ARRAY_TASK_ID ;rc=$?
+python call_python_script.py $SLURM_ARRAY_TASK_ID ;rc=$?
end=`date`
echo "Finished task $SLURM_ARRAY_TASK_ID with exit code $rc at $end"
diff --git a/examples/call_python_script.py b/examples/call_python_script.py
new file mode 100644
index 0000000..ee5c6c2
--- /dev/null
+++ b/examples/call_python_script.py
@@ -0,0 +1,100 @@
+import os, sys, subprocess, threading, time
+
+class CommandExecution(object):
+ """ Represents the execution of a single command line argument. """
+ def __init__(self):
+ self.timelimit = None
+ self.return_code = None
+ self.output = None
+ self.wall_time = None
+ self.proc = None
+
+ def stop(self):
+ self.timelimit = True
+ self.proc.kill()
+
+ def run(self, command_line_str, timelimit):
+ command_line_list = command_line_str.split()
+ command_line_list[0] = os.path.expanduser(command_line_list[0])
+ self.proc = subprocess.Popen(command_line_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ start_time = time.time()
+ timer = threading.Timer(timelimit, self.stop)
+ self.timelimit = False
+ self.output = ""
+ timer.start()
+ try:
+ stdout, stderr = self.proc.communicate()
+ except Exception as e:
+ self.output = self.output + "Error when executing the command:\n{}\n".format(e)
+ finally:
+ timer.cancel()
+ self.wall_time = time.time() - start_time
+ self.return_code = self.proc.returncode
+ self.output = self.output + stdout.decode('utf8')
+ if len(stderr) > 0:
+ self.output = self.output + "\n" + "#"*30 + "Output to stderr" + "#"*30 + "\n" + stderr.decode('utf8')
+ if self.timelimit and self.wall_time <= timelimit:
+ print("WARN: A timelimit was triggered although the measured time is {} seconds which is still below the time limit of {} seconds".format(self.wall_time, timelimit))
+
+
+def execute_command_line(command_line_str, timelimit):
+ """
+ Executes the given command line with the given time limit (in seconds).
+ :returns the output of the command (including the output to stderr, if present), the runtime of the command and either the return code or None (in case of a timelimit)
+ """
+ execution = CommandExecution()
+ execution.run(command_line_str, timelimit)
+ if execution.timelimit:
+ return execution.output, execution.wall_time, None
+ else:
+ return execution.output, execution.wall_time, execution.return_code
+
+
+def execute_and_log_command_line(command_line_str, timelimit, logfile):
+ """
+ Executes the given command line with the given time limit (in seconds) and stores the output the provided logfile.
+ :returns a string reporting the runtime, returncode and logfile location.
+ """
+ output, runtime, returncode = execute_command_line(command_line_str, timelimit)
+ header = "COMMAND: {}\nWALLTIME (seconds): {}\n".format(command_line_str, runtime)
+ if returncode == 0:
+ message = "done after {} seconds. Output saved to {}".format(runtime, logfile)
+ header += "\n"
+ elif returncode is None and runtime >= timelimit:
+ message = "timeout after {} seconds. Output saved to {}".format(runtime, logfile)
+ header += "TIMEOUT\n"
+ else:
+ message = "error after {} seconds with non-zero exit code {}. Output saved to {}".format(runtime,returncode,logfile)
+ header += "ERROR: {}\n".format(returncode)
+ output = header + "#"*30 + " LOG " + "#"*30 + "\n" + output
+ with open(logfile, 'w') as file:
+ file.write(output)
+ return message
+
+timelimit = 3600
+logdir = "/rwthfs/rz/cluster/home/tq429871/logs/"
+
+# contains (logfilename, command)-pairs
+commands = []
+commands.append((r"eajs_P2-U3_pos-t8-c1-p0.01-e0.log", r"/home/tq429871/storm/build/bin/storm --prism /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/eajs/eajs02.prism --prop /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/eajs/eajs.props -const U=3 --multiobjective:schedrest positional -tm --exact --lpsolver gurobi --multiobjective:printres --gurobi:threads 8 --gurobi:concurrentmip 1 --multiobjective:precision 0.01 --multiobjective:maxsteps 0"))
+commands.append((r"resource-gathering_G10-E5-S100_pos-t1-c1-p0.01-e24.log", r"/home/tq429871/storm/build/bin/storm --prism /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/resource-gathering/resource-gathering.prism --prop /rwthfs/rz/cluster/home/tq429871/git/pure-strategies-benchmarks/scripts/../models/resource-gathering/resource-gathering.props -const REQGOLD=10,REQGEM=5,AVSTEPS=100 --multiobjective:schedrest positional -tm --exact --lpsolver gurobi --multiobjective:printres --gurobi:threads 1 --gurobi:concurrentmip 1 --multiobjective:precision 0.01 --multiobjective:maxsteps 24"))
+
+if __name__ == "__main__":
+ print("Found {} commands.".format(len(commands)))
+ if len(sys.argv) != 2:
+ print("Invalid number of arguments. Expected 1 but got {}. Usage\npython {} <id>".format(len(sys.argv) - 1, sys.argv[0]))
+ exit(1)
+ try:
+ id = int(sys.argv[1])
+ except ValueError:
+ print("Unable to convert argument '{}' to an id".format(sys.argv[1]))
+ exit(1)
+ if id < 0 or id >= len(commands):
+ print("id {} is out of range. Should be between 0 and {}".format(id, len(commands) - 1))
+ exit(1)
+
+ sys.stdout.write("Command {}/{} ... ".format(id, len(commands) - 1))
+ sys.stdout.flush()
+ logfile = os.path.join(logdir, commands[id][0])
+ message = execute_and_log_command_line(commands[id][1], timelimit, logfile)
+ print(message)
--
GitLab