Commit b766f0ca authored by Lukas Weber's avatar Lukas Weber
Browse files

making vector_length of observables constant (breaking change) improving

cli
parent 67b338aa
......@@ -6,9 +6,9 @@ import subprocess
from loadleveller import jobfile
import os
parser = argparse.ArgumentParser(description='Helper script for running and managing loadleveller Monte Carlo jobs.', usage='''loadl <command> <jobfile> [<args>]
parser = argparse.ArgumentParser(description='Helper script for running and managing loadleveller Monte Carlo jobs.', usage='''loadl <command> <jobscript> [<args>]
<jobfile> is a script file containing the parameters for the job of interest.
<jobscript> is an executable that prints the job parameter YAML-file to stdout. It is convenient to use the taskmaker python module for this purpose.
Possible commands and their shorthands are
delete, d delete all data related to a job
......@@ -19,25 +19,30 @@ Possible commands and their shorthands are
parser.add_argument('command')
parser.add_argument('jobfile')
args = parser.parse_args(sys.argv[1:3])
args, leftover_args = parser.parse_known_args()
# all paths are relative to the jobscript
jobdir = os.path.dirname(args.jobfile)
jobfile_name = os.path.basename(args.jobfile)
job = jobfile.JobFile(jobfile_name)
if jobdir != '':
print('$ cd '+jobdir)
os.chdir(jobdir)
try:
job = jobfile.JobFile('./'+os.path.basename(args.jobfile))
except jobfile.JobFileGenError as e:
print('Error: {}'.format(e))
sys.exit(1)
def run():
import glob
from loadleveller import clusterutils
parser = argparse.ArgumentParser(description='run a loadleveller job on a cluster or locally')
parser.add_argument('-s','--single', action='store_true', help='Run in the single core scheduler mode')
parser.add_argument('-f', '--force', action='store_true', help='Ignore warnings about possible job corruption')
parser.add_argument('-r', '--restart', action='store_true', help='Delete all existing job data before starting.')
args_run = parser.parse_args(sys.argv[3:])
parser.add_argument('-s','--single', action='store_true', help='run in the single core scheduler mode')
parser.add_argument('-f', '--force', action='store_true', help='ignore warnings about possible job corruption')
parser.add_argument('-r', '--restart', action='store_true', help='delete all existing job data before starting.')
args_run = parser.parse_args(leftover_args)
if args_run.restart:
delete()
......@@ -57,12 +62,13 @@ def run():
except StopIteration:
pass
job_input_filename = job.write_job_input_file()
if args_run.single:
cmd = [job.jobconfig['mc_binary'], 'single', jobfile_name]
cmd = [job.jobconfig['mc_binary'], 'single', job_input_filename]
print('$ '+' '.join(cmd))
subprocess.run(cmd)
else:
clusterutils.run(job.jobname, job.jobconfig, [job.jobconfig['mc_binary'], jobfile_name])
clusterutils.run(job.jobname, job.jobconfig, [job.jobconfig['mc_binary'], job_input_filename])
def delete():
import shutil
......@@ -77,13 +83,14 @@ def delete():
os.unlink(results_file)
def merge():
cmd = [job.jobconfig['mc_binary'], 'merge', jobfile_name]
job_input_filename = job.write_job_input_file()
cmd = [job.jobconfig['mc_binary'], 'merge', job_input_filename]
print('$ '+' '.join(cmd))
subprocess.run(cmd)
def status():
from loadleveller import jobstatus
rc = jobstatus.print_status(job, sys.argv[3:])
rc = jobstatus.print_status(job, leftover_args)
sys.exit(rc)
......
import yaml
import os
import subprocess
'''Helpers for handling loadleveller jobfiles.'''
'''Helpers for handling loadleveller jobfiles/scripts. For lack of a better idea, the job description files of loadleveller are actually executables that output a more verbose yaml parameter file to stdout. Use the taskmaker module to write the input scripts.'''
class JobFileGenError(Exception):
pass
class JobFile:
def __init__(self, filename):
with open(filename, 'r') as f:
jobfile = yaml.safe_load(f)
self.__dict__ = jobfile
result = subprocess.run([filename], stdout=subprocess.PIPE)
self.raw_jobfile = result.stdout.decode('utf-8')
if result.returncode != 0:
raise JobFileGenError('Generation script "{}" had a non-zero return code. Treating as error.'.format(filename))
try:
parsed_job = yaml.safe_load(self.raw_jobfile)
self.__dict__.update(parsed_job)
except Exception as e:
raise JobFileGenError('Could not parse job generation script output: {}'.format(e))
def write_job_input_file(self):
try:
datadir = self.jobname + '.data'
try:
os.makedirs(datadir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
job_input_filename = os.path.join(datadir, 'parameters.yaml')
with open(job_input_filename, 'w') as f:
f.write(self.raw_jobfile)
except Exception as e:
raise JobFileGenError('Could not write parameters.yaml: {}'.format(e))
return job_input_filename
......@@ -4,9 +4,9 @@
namespace loadl {
observable::observable(std::string name, size_t bin_length, size_t vector_length)
: name_{std::move(name)}, bin_length_{bin_length}, vector_length_{vector_length},
current_bin_{0}, current_bin_filling_{0} {
: name_{std::move(name)}, bin_length_{bin_length}, vector_length_{vector_length} {
samples_.reserve(vector_length_ * initial_bin_length);
samples_.resize(vector_length_);
}
const std::string &observable::name() const {
......@@ -20,32 +20,31 @@ void observable::checkpoint_write(const iodump::group &dump_file) const {
// So if current_bin_ is not 0 here, we have made a mistake.
assert(current_bin_ == 0);
// Another sanity check: if there is a partial bin, the samples_ array should contain it.
assert(current_bin_filling_ > 0 && samples_.size() == 1);
// Another sanity check: the samples_ array should contain one partial bin.
assert(samples_.size() == vector_length_);
dump_file.write("name", name_);
dump_file.write("vector_length", vector_length_);
dump_file.write("bin_length", bin_length_);
dump_file.write("initial_length", initial_length_);
dump_file.write("current_bin_filling", current_bin_filling_);
dump_file.write("samples", samples_);
}
void observable::measurement_write(const iodump::group &meas_file) {
std::vector<double> current_bin_value;
// if there is at least one bin…
if(vector_length_ > 0 && samples_.size() >= vector_length_) {
current_bin_value.assign(samples_.end() - vector_length_, samples_.end());
if(samples_.size() > vector_length_) {
std::vector<double> current_bin_value(samples_.end() - vector_length_, samples_.end());
samples_.resize(current_bin_ * vector_length_);
meas_file.insert_back("samples", samples_);
samples_ = current_bin_value;
assert(samples_.size() == vector_length_);
} else {
meas_file.insert_back("samples", std::vector<double>()); // just touch
}
meas_file.write("vector_length", vector_length_);
meas_file.write("bin_length", bin_length_);
meas_file.insert_back("samples", samples_);
samples_ = current_bin_value;
samples_.reserve(initial_length_ * vector_length_);
current_bin_ = 0;
}
......@@ -53,7 +52,6 @@ void observable::checkpoint_read(const iodump::group &d) {
d.read("name", name_);
d.read("vector_length", vector_length_);
d.read("bin_length", bin_length_);
d.read("initial_length", initial_length_);
d.read("current_bin_filling", current_bin_filling_);
d.read("samples", samples_);
current_bin_ = 0;
......
......@@ -10,7 +10,7 @@ namespace loadl {
class observable {
public:
observable(std::string name, size_t bin_length = 1, size_t vector_length = 1);
observable(std::string name, size_t bin_length, size_t vector_length);
const std::string &name() const;
......@@ -31,11 +31,10 @@ private:
static const size_t initial_bin_length = 1000;
std::string name_;
size_t bin_length_;
size_t vector_length_;
size_t initial_length_;
size_t current_bin_;
size_t current_bin_filling_;
size_t bin_length_{};
size_t vector_length_{};
size_t current_bin_{};
size_t current_bin_filling_{};
std::vector<double> samples_;
};
......@@ -48,13 +47,9 @@ void observable::add(T val) {
template<class T>
void observable::add(const std::vector<T> &val) {
// handle wrong vector length gracefully on first add
if(current_bin_ == 0 && current_bin_filling_ == 0) {
vector_length_ = val.size();
samples_.resize((current_bin_ + 1) * vector_length_, 0.);
} else if(vector_length_ != val.size()) {
if(vector_length_ != val.size()) {
throw std::runtime_error{fmt::format(
"observable::add: added vector has different size ({}) than what was added before ({})",
"observable::add: added vector has unexpected size ({}). Observable was initialized with vector length ({})",
val.size(), vector_length_)};
}
......@@ -63,9 +58,11 @@ void observable::add(const std::vector<T> &val) {
current_bin_filling_++;
if(current_bin_filling_ == bin_length_) { // need to start a new bin next time
if(bin_length_ > 1)
for(size_t j = 0; j < vector_length_; ++j)
if(bin_length_ > 1) {
for(size_t j = 0; j < vector_length_; ++j) {
samples_[current_bin_ * vector_length_ + j] /= bin_length_;
}
}
current_bin_++;
samples_.resize((current_bin_ + 1) * vector_length_);
current_bin_filling_ = 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment