Commit 9760a689 authored by Lukas Weber's avatar Lukas Weber

adding proper python module/more meson

parent a4fb3a18
......@@ -3,7 +3,6 @@ project('load-leveller', 'c', 'cpp',
meson_version : '>=0.50.0',
default_options : [
'warning_level=3',
'cpp_std=c++17',
]
)
......@@ -11,11 +10,12 @@ fmt_dep = dependency('fmt', fallback : ['fmt', 'fmt_dep'])
yamlcpp_dep = dependency('yaml-cpp', version : '>=0.6.0', fallback : ['yaml-cpp', 'yamlcpp_dep'])
mpi_dep = dependency('mpi')
# HDF5 is a pain
cc = meson.get_compiler('cpp')
hdf5_lib = cc.find_library('hdf5')
should_install = meson.is_subproject()
loadleveller_deps = [ fmt_dep, yamlcpp_dep, mpi_dep, hdf5_lib ]
subdir('src')
build
dist
loadleveller.egg-info
Copyright (c) 2018 The Python Packaging Authority
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
import yaml
import numpy as np
import itertools
'''This module can be used to easily extract Monte Carlo results from the *.results.yml file produced by the loadleveller library.'''
class Observable:
def __init__(self, num_tasks):
self.rebinning_bin_length = np.zeros(num_tasks)
self.rebinning_bin_count = np.zeros(num_tasks)
self.autocorrelation_time = np.zeros(num_tasks)+np.nan
# have to be set later because dimension is unknown
self.mean = None
self.error = None
class MCArchive:
def __init__(self, filename):
with open(filename, 'r') as f:
doc = yaml.load(f)
param_names = set(sum([list(task['parameters'].keys()) for task in doc], []))
observable_names = set(sum([list(task['results'].keys()) for task in doc], []))
self.num_tasks = len(doc)
self.parameters = dict(zip(param_names, [[None for _ in range(self.num_tasks)] for _ in param_names]))
self.observables = dict(zip(observable_names, [Observable(self.num_tasks) for _ in observable_names]))
for i, task in enumerate(doc):
for param, value in task['parameters'].items():
self.parameters[param][i] = value
for obs, value in task['results'].items():
o = self.observables[obs]
o.rebinning_bin_length[i] = value['rebinning_bin_length']
o.rebinning_bin_count[i] = value['rebinning_bin_count']
o.autocorrelation_time[i] = value['autocorrelation_time']
# fill in dimensions on first occurence of the observable
if np.all(o.mean == None):
o.mean = np.zeros([self.num_tasks, len(value['mean'])]) + np.nan
o.error = np.zeros([self.num_tasks, len(value['error'])]) + np.nan
if o.mean.shape != o.error.shape:
raise 'observable "{}": dimension mismatch between mean and error'.format(obs)
else:
if len(value['mean']) != o.mean.shape[1] or len(value['error']) != o.error.shape[1]:
raise 'observable "{}": dimension mismatch between different tasks'.format(obs)
o.mean[i,:] = value['mean']
o.error[i,:] = value['error']
def condition_mask(self, conditions):
if not conditions:
return [True for _ in range(self.num_tasks)]
return [all(self.parameters[key][i] == val for key, val in conditions.items()) for i in range(self.num_tasks)]
def get_parameter(self, name, unique=False, conditions={}):
selection = list(itertools.compress(self.parameters[name], self.condition_mask(conditions)))
if unique:
return list(sorted(set(selection)))
return selection
def get_observable(self, name, conditions={}):
orig = self.observables[name]
selection = Observable(0)
mask = self.condition_mask(conditions)
selection.rebinning_bin_count = orig.rebinning_bin_count[mask]
selection.rebinning_bin_length = orig.rebinning_bin_length[mask]
selection.autocorrelation_time = orig.autocorrelation_time[mask]
selection.mean = orig.mean[mask,:]
selection.error = orig.error[mask,:]
return selection
import setuptools
setuptools.setup(
name="loadleveller",
version="2.0.0",
author="Lukas Weber",
author_email="lweber@physik.rwth-aachen.de",
description="Python tools for the loadleveller library",
url="https://git.rwth-aachen.de/lukas.weber2/load_leveller",
packages=setuptools.find_packages(),
license="MIT",
scripts=["ydelete","yrun","ystatus"],
install_requires=["pyyaml","h5py","numpy"]
)
......@@ -8,7 +8,7 @@ import glob
import sys
import os
parser = argparse.ArgumentParser(description='This helper program runs a load-leveller Monte Carlo program using a provided YAML-formatted jobfile. The jobfile contains information on how to run the job (what mc binary, mpi-parameters, ...) and a number of tasks with different simulation parameters each. When running on a cluster batch system, the batch script is generated using ygeneratebatchscript.')
parser = argparse.ArgumentParser(description='This helper program runs a loadleveller Monte Carlo program using a provided YAML-formatted jobfile. The jobfile contains information on how to run the job (what mc binary, mpi-parameters, ...) and a number of tasks with different simulation parameters each. When running on a cluster batch system, the batch script is generated using ygeneratebatchscript.')
parser.add_argument('jobfile', metavar='JOBFILE', help='Configuration file containing all the job information. May be generated using ytaskmaker')
parser.add_argument('-s','--single', action='store_true', help='Run in the single core scheduler mode')
......@@ -41,12 +41,13 @@ try:
error = False
label = 'Warning' if args.force else 'Error'
if binary_modtime > data_modtime:
print('{}: binary \'{}\' is newer than the checkpoint files.\nUse ydelete to start from a blank run or use \'--force\' to proceed if you are sure\nthe new binary is compatible.'.format(label, mcbinary))
print('{}: binary \'{}\' is newer than the checkpoint files.'.format(label, mcbinary))
error = True
if jobfile_modtime > data_modtime:
print('{}: jobfile \'{}\' is newer than the checkpoint files.\nStarting the run now might lead to corruption. Use ydelete to start\nfrom the blank run or use \'--force\' if you are very sure.'.format(label, args.jobfile))
print('{}: jobfile \'{}\' is newer than the checkpoint files.'.format(label, args.jobfile))
error = True
if not args.force and error:
print('Use ydelete to start from a blank run or use \'--force\' to proceed if you are sure\nthe changes you made are compatible.')
sys.exit(1)
except StopIteration:
pass
......
#!/usr/bin/env python3
import argparse
import glob
import h5py
import os
import yaml
parser = argparse.ArgumentParser(description='Prints the status and progress of a loadleveller Monte Carlo job.')
parser.add_argument('jobfile', metavar='JOBFILE', help='Configuration file containing all the job information.')
args = parser.parse_args()
with open(args.jobfile, 'r') as f:
jobfile = yaml.load(f)
tasks = jobfile['tasks'].keys()
needs_restart = False
needs_merge = False
result_mtime = 0
try:
result_mtime = os.path.getmtime(args.jobfile+'.results.yml')
except(FileNotFoundError):
pass
for task in tasks:
target_sweeps = jobfile['tasks'][task]['sweeps']
target_therm = jobfile['tasks'][task]['thermalization']
sweeps = 0
therm_sweeps = 0
num_runs = 0
for runfile in glob.iglob('{}.data/{}/run*.dump.h5'.format(args.jobfile,task)):
num_runs += 1
with h5py.File(runfile, 'r') as f:
sweeps += f['/sweeps'][0]
therm_sweeps += f['/thermalization_sweeps'][0]
for measfile in glob.iglob('{}.data/{}/run*.meas.h5'.format(args.jobfile,task)):
if os.path.getmtime(measfile) > result_mtime:
needs_merge = True
if sweeps < target_sweeps + target_therm:
needs_restart = True
print('{}: {} runs, {}/{} sweeps, {}/{} thermalization'.format(task, num_runs, sweeps-therm_sweeps, target_sweeps, therm_sweeps, target_therm))
if needs_restart:
print('\nNeeds restart!')
if needs_merge:
print('\nNeeds merge! (run \'yrun -m {0}\' to get an up-to-date {0}.results.yml file)'.format(args.jobfile))
......@@ -167,12 +167,12 @@ iodump::h5_handle iodump::group::create_dataset(const std::string &name, hid_t d
h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
hsize_t oldsize = H5Sget_simple_extent_npoints(*dataspace);
hssize_t oldsize = H5Sget_simple_extent_npoints(*dataspace);
if(oldsize < 0) {
throw iodump_exception{"H5Sget_simple_extent_npoints"};
}
if(oldsize != size) {
if(static_cast<hsize_t>(oldsize) != size) {
throw std::runtime_error{
"iodump: tried to write into an existing dataset with different dimensions!"};
}
......
......@@ -5,6 +5,8 @@ abstract_mc::abstract_mc(const std::string &jobfile, const std::string &taskname
therm_ = param.get<int>("thermalization");
}
void abstract_mc::write_output(const std::string &) {}
void abstract_mc::random_init() {
if(param.defined("seed")) {
rng.reset(new randomnumbergenerator(param.get<uint64_t>("seed")));
......
......@@ -22,7 +22,7 @@ protected:
virtual void init() = 0;
virtual void checkpoint_write(const iodump::group &out) = 0;
virtual void checkpoint_read(const iodump::group &in) = 0;
virtual void write_output(const std::string &filename){};
virtual void write_output(const std::string &filename);
virtual void do_update() = 0;
virtual void do_measurement() = 0;
......
......@@ -34,16 +34,18 @@ libloadleveller = library('loadleveller',
loadleveller_sources,
dependencies : loadleveller_deps,
version : meson.project_version(),
install : true,
install : should_install,
override_options : ['c_std=none']
)
pkg = import('pkgconfig')
pkg.generate(libloadleveller,
description : 'Framework for distributed (Quantum) Monte Carlo codes',
)
install_headers(loadleveller_headers, subdir : 'loadleveller')
if should_install
pkg = import('pkgconfig')
pkg.generate(libloadleveller,
description : 'Framework for distributed (Quantum) Monte Carlo codes',
)
install_headers(loadleveller_headers, subdir : 'loadleveller')
endif
loadleveller_dep = declare_dependency(
include_directories : include_directories('../include/'),
......
......@@ -245,23 +245,6 @@ void runner_master::read() {
}
void runner_master::end_of_run() {
bool need_restart = false;
for(size_t i = 0; i < tasks_.size(); i++) {
if(!tasks_[i].is_done()) {
need_restart = true;
break;
}
}
if(need_restart) {
std::string rfilename = job_.jobfile_name + ".restart";
std::ofstream rfile(rfilename);
rfile << "restart me\n";
rfile.close();
job_.status << "0 : Restart needed"
<< "\n";
}
report();
}
......
......@@ -7,7 +7,7 @@
#include <iostream>
#include <sys/stat.h>
int runner_single_start(jobinfo job, const mc_factory &mccreator, int argc, char **argv) {
int runner_single_start(jobinfo job, const mc_factory &mccreator, int, char **) {
runner_single r{std::move(job), mccreator};
r.start();
return 0;
......@@ -94,21 +94,6 @@ void runner_single::read() {
}
void runner_single::end_of_run() {
bool need_restart = false;
for(size_t i = 0; i < tasks_.size(); i++) {
if(!tasks_[i].is_done()) {
need_restart = true;
break;
}
}
if(need_restart) {
std::string rfilename = job_.jobfile_name + ".restart";
std::ofstream rfile(rfilename.c_str());
rfile << "restart me\n";
rfile.close();
job_.status << 0 << " : Restart needed"
<< "\n";
}
report();
job_.status << 0 << " : finalized"
<< "\n";
......
......@@ -7,5 +7,5 @@ runner_task::runner_task(int target_sweeps, int target_thermalization, int sweep
scheduled_runs{scheduled_runs} {}
bool runner_task::is_done() const {
return sweeps > (target_sweeps + target_thermalization);
return sweeps >= (target_sweeps + target_thermalization);
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment