adding proper python module/more meson

meson.build

@@ -3,7 +3,6 @@ project('load-leveller', 'c', 'cpp',
   meson_version : '>=0.50.0',
   default_options : [
     'warning_level=3',
-    'cpp_std=c++17',
   ]
 )
 
@@ -11,11 +10,12 @@ fmt_dep = dependency('fmt', fallback : ['fmt', 'fmt_dep'])
 yamlcpp_dep = dependency('yaml-cpp', version : '>=0.6.0', fallback : ['yaml-cpp', 'yamlcpp_dep'])
 mpi_dep = dependency('mpi')
 
-
 # HDF5 is a pain
 cc = meson.get_compiler('cpp')
 hdf5_lib = cc.find_library('hdf5')
 
+should_install = meson.is_subproject()
+
 loadleveller_deps = [ fmt_dep, yamlcpp_dep, mpi_dep, hdf5_lib ]
 
 subdir('src')

python/.gitignore

+build
+dist
+loadleveller.egg-info
+

python/LICENSE

+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python/loadleveller/mcextract.py

+import yaml
+import numpy as np
+import itertools
+
+'''This module can be used to easily extract Monte Carlo results from the *.results.yml file produced by the loadleveller library.'''
+
+class Observable:
+    def __init__(self, num_tasks):
+        self.rebinning_bin_length = np.zeros(num_tasks)
+        self.rebinning_bin_count = np.zeros(num_tasks)
+        self.autocorrelation_time = np.zeros(num_tasks)+np.nan
+
+        # have to be set later because dimension is unknown
+        self.mean = None
+        self.error = None        
+
+class MCArchive:
+    def __init__(self, filename):
+        with open(filename, 'r') as f:
+            doc = yaml.load(f)
+
+        param_names = set(sum([list(task['parameters'].keys()) for task in doc], []))
+        observable_names = set(sum([list(task['results'].keys()) for task in doc], []))
+        self.num_tasks = len(doc)
+        
+        self.parameters = dict(zip(param_names, [[None for _ in range(self.num_tasks)] for _ in param_names]))
+        self.observables = dict(zip(observable_names, [Observable(self.num_tasks) for _ in observable_names]))
+
+        for i, task in enumerate(doc):
+            for param, value in task['parameters'].items():
+                self.parameters[param][i] = value
+
+            for obs, value in task['results'].items():
+                o = self.observables[obs]
+                o.rebinning_bin_length[i] = value['rebinning_bin_length']
+                o.rebinning_bin_count[i] = value['rebinning_bin_count']
+                o.autocorrelation_time[i] = value['autocorrelation_time']
+
+                # fill in dimensions on first occurence of the observable
+                if np.all(o.mean == None):
+                    o.mean = np.zeros([self.num_tasks, len(value['mean'])]) + np.nan
+                    o.error = np.zeros([self.num_tasks, len(value['error'])]) + np.nan
+
+                    if o.mean.shape != o.error.shape:
+                        raise 'observable "{}": dimension mismatch between mean and error'.format(obs)
+                else:
+                    if len(value['mean']) != o.mean.shape[1] or len(value['error']) != o.error.shape[1]:
+                        raise 'observable "{}": dimension mismatch between different tasks'.format(obs)
+
+                o.mean[i,:] = value['mean']
+                o.error[i,:] = value['error']
+
+    def condition_mask(self, conditions):
+        if not conditions:
+            return [True for _ in range(self.num_tasks)]
+
+        return [all(self.parameters[key][i] == val for key, val in conditions.items()) for i in range(self.num_tasks)]
+
+    def get_parameter(self, name, unique=False, conditions={}):
+        selection = list(itertools.compress(self.parameters[name], self.condition_mask(conditions)))
+        
+        if unique:
+            return list(sorted(set(selection)))
+        return selection
+
+    def get_observable(self, name, conditions={}):
+        orig = self.observables[name]
+        
+        selection = Observable(0)
+        mask = self.condition_mask(conditions)
+        selection.rebinning_bin_count = orig.rebinning_bin_count[mask]
+        selection.rebinning_bin_length = orig.rebinning_bin_length[mask]
+        selection.autocorrelation_time = orig.autocorrelation_time[mask]
+        selection.mean = orig.mean[mask,:]
+        selection.error = orig.error[mask,:]
+
+        return selection

python/setup.py

+import setuptools
+
+setuptools.setup(
+    name="loadleveller",
+    version="2.0.0",
+    author="Lukas Weber",
+    author_email="lweber@physik.rwth-aachen.de",
+    description="Python tools for the loadleveller library",
+    url="https://git.rwth-aachen.de/lukas.weber2/load_leveller",
+    packages=setuptools.find_packages(),
+    license="MIT",
+    scripts=["ydelete","yrun","ystatus"],
+    install_requires=["pyyaml","h5py","numpy"]
+)

tools/yrun → python/yrun

@@ -8,7 +8,7 @@ import glob
 import sys
 import os
 
-parser = argparse.ArgumentParser(description='This helper program runs a load-leveller Monte Carlo program using a provided YAML-formatted jobfile. The jobfile contains information on how to run the job (what mc binary, mpi-parameters, ...) and a number of tasks with different simulation parameters each. When running on a cluster batch system, the batch script is generated using ygeneratebatchscript.')
+parser = argparse.ArgumentParser(description='This helper program runs a loadleveller Monte Carlo program using a provided YAML-formatted jobfile. The jobfile contains information on how to run the job (what mc binary, mpi-parameters, ...) and a number of tasks with different simulation parameters each. When running on a cluster batch system, the batch script is generated using ygeneratebatchscript.')
 
 parser.add_argument('jobfile', metavar='JOBFILE', help='Configuration file containing all the job information. May be generated using ytaskmaker')
 parser.add_argument('-s','--single', action='store_true', help='Run in the single core scheduler mode')
@@ -41,12 +41,13 @@ try:
     error = False
     label = 'Warning' if args.force else 'Error'
     if binary_modtime > data_modtime:
-        print('{}: binary \'{}\' is newer than the checkpoint files.\nUse ydelete to start from a blank run or use \'--force\' to proceed if you are sure\nthe new binary is compatible.'.format(label, mcbinary))
+        print('{}: binary \'{}\' is newer than the checkpoint files.'.format(label, mcbinary))
         error = True
     if jobfile_modtime > data_modtime:
-        print('{}: jobfile \'{}\' is newer than the checkpoint files.\nStarting the run now might lead to corruption. Use ydelete to start\nfrom the blank run or use \'--force\' if you are very sure.'.format(label, args.jobfile))
+        print('{}: jobfile \'{}\' is newer than the checkpoint files.'.format(label, args.jobfile))
         error = True
     if not args.force and error:
+        print('Use ydelete to start from a blank run or use \'--force\' to proceed if you are sure\nthe changes you made are compatible.')
         sys.exit(1)
 except StopIteration:
     pass

python/ystatus

+#!/usr/bin/env python3
+
+import argparse
+import glob
+import h5py
+import os
+import yaml
+
+parser = argparse.ArgumentParser(description='Prints the status and progress of a loadleveller Monte Carlo job.')
+parser.add_argument('jobfile', metavar='JOBFILE', help='Configuration file containing all the job information.')
+
+args = parser.parse_args()
+
+with open(args.jobfile, 'r') as f:
+    jobfile = yaml.load(f)
+
+tasks = jobfile['tasks'].keys()
+
+needs_restart = False
+needs_merge = False
+
+result_mtime = 0
+try:
+    result_mtime = os.path.getmtime(args.jobfile+'.results.yml')
+except(FileNotFoundError):
+    pass
+
+for task in tasks:
+    target_sweeps = jobfile['tasks'][task]['sweeps']
+    target_therm = jobfile['tasks'][task]['thermalization']
+
+    sweeps = 0
+    therm_sweeps = 0
+
+    num_runs = 0
+    
+    for runfile in glob.iglob('{}.data/{}/run*.dump.h5'.format(args.jobfile,task)):
+        num_runs += 1
+
+        with h5py.File(runfile, 'r') as f:
+            sweeps += f['/sweeps'][0]
+            therm_sweeps += f['/thermalization_sweeps'][0]
+
+    for measfile in glob.iglob('{}.data/{}/run*.meas.h5'.format(args.jobfile,task)):
+        if os.path.getmtime(measfile) > result_mtime:
+            needs_merge = True
+
+    if sweeps < target_sweeps + target_therm:
+        needs_restart = True
+
+    print('{}: {} runs, {}/{} sweeps, {}/{} thermalization'.format(task, num_runs, sweeps-therm_sweeps, target_sweeps, therm_sweeps, target_therm))
+
+if needs_restart:
+    print('\nNeeds restart!')
+if needs_merge:
+    print('\nNeeds merge! (run \'yrun -m {0}\' to get an up-to-date {0}.results.yml file)'.format(args.jobfile))

src/dump.cpp

@@ -167,12 +167,12 @@ iodump::h5_handle iodump::group::create_dataset(const std::string &name, hid_t d
 		h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
 		h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
 
-		hsize_t oldsize = H5Sget_simple_extent_npoints(*dataspace);
+		hssize_t oldsize = H5Sget_simple_extent_npoints(*dataspace);
 
 		if(oldsize < 0) {
 			throw iodump_exception{"H5Sget_simple_extent_npoints"};
 		}
-		if(oldsize != size) {
+		if(static_cast<hsize_t>(oldsize) != size) {
 			throw std::runtime_error{
 			    "iodump: tried to write into an existing dataset with different dimensions!"};
 		}

src/mc.cpp

@@ -5,6 +5,8 @@ abstract_mc::abstract_mc(const std::string &jobfile, const std::string &taskname
 	therm_ = param.get<int>("thermalization");
 }
 
+void abstract_mc::write_output(const std::string &) {}
+
 void abstract_mc::random_init() {
 	if(param.defined("seed")) {
 		rng.reset(new randomnumbergenerator(param.get<uint64_t>("seed")));

src/mc.h

@@ -22,7 +22,7 @@ protected:
 	virtual void init() = 0;
 	virtual void checkpoint_write(const iodump::group &out) = 0;
 	virtual void checkpoint_read(const iodump::group &in) = 0;
-	virtual void write_output(const std::string &filename){};
+	virtual void write_output(const std::string &filename);
 	virtual void do_update() = 0;
 	virtual void do_measurement() = 0;
 

src/meson.build

@@ -34,16 +34,18 @@ libloadleveller = library('loadleveller',
   loadleveller_sources,
   dependencies : loadleveller_deps,
   version : meson.project_version(),
-  install : true,
+  install : should_install,
   override_options : ['c_std=none']
 )
 
-pkg = import('pkgconfig')
-pkg.generate(libloadleveller,
-  description : 'Framework for distributed (Quantum) Monte Carlo codes',
-)
-
-install_headers(loadleveller_headers, subdir : 'loadleveller')
+if should_install
+  pkg = import('pkgconfig')
+  pkg.generate(libloadleveller,
+    description : 'Framework for distributed (Quantum) Monte Carlo codes',
+  )
+  
+  install_headers(loadleveller_headers, subdir : 'loadleveller')
+endif
 
 loadleveller_dep = declare_dependency(
   include_directories : include_directories('../include/'),

src/runner.cpp

@@ -245,23 +245,6 @@ void runner_master::read() {
 }
 
 void runner_master::end_of_run() {
-	bool need_restart = false;
-	for(size_t i = 0; i < tasks_.size(); i++) {
-		if(!tasks_[i].is_done()) {
-			need_restart = true;
-			break;
-		}
-	}
-
-	if(need_restart) {
-		std::string rfilename = job_.jobfile_name + ".restart";
-		std::ofstream rfile(rfilename);
-		rfile << "restart me\n";
-		rfile.close();
-		job_.status << "0 : Restart needed"
-		            << "\n";
-	}
-
 	report();
 }
 

src/runner_single.cpp

@@ -7,7 +7,7 @@
 #include <iostream>
 #include <sys/stat.h>
 
-int runner_single_start(jobinfo job, const mc_factory &mccreator, int argc, char **argv) {
+int runner_single_start(jobinfo job, const mc_factory &mccreator, int, char **) {
 	runner_single r{std::move(job), mccreator};
 	r.start();
 	return 0;
@@ -94,21 +94,6 @@ void runner_single::read() {
 }
 
 void runner_single::end_of_run() {
-	bool need_restart = false;
-	for(size_t i = 0; i < tasks_.size(); i++) {
-		if(!tasks_[i].is_done()) {
-			need_restart = true;
-			break;
-		}
-	}
-	if(need_restart) {
-		std::string rfilename = job_.jobfile_name + ".restart";
-		std::ofstream rfile(rfilename.c_str());
-		rfile << "restart me\n";
-		rfile.close();
-		job_.status << 0 << " : Restart needed"
-		            << "\n";
-	}
 	report();
 	job_.status << 0 << " : finalized"
 	            << "\n";

src/runner_task.cpp

@@ -7,5 +7,5 @@ runner_task::runner_task(int target_sweeps, int target_thermalization, int sweep
       scheduled_runs{scheduled_runs} {}
 
 bool runner_task::is_done() const {
-	return sweeps > (target_sweeps + target_thermalization);
+	return sweeps >= (target_sweeps + target_thermalization);
 }