yaml parser replacement

definitions.h

-#pragma once
-
-enum {
-	MASTER = 0,
-	T_STATUS = 1,
-	T_ACTION = 2,
-	T_NEW_JOB = 3,
-	T_PARTNER = 4,
-	T_WEIGHT = 5,
-	T_LABEL = 6,
-	T_POS = 7,
-
-	S_IDLE = 1,
-	S_BUSY = 2,
-	S_FINISHED = 3,
-	S_GLOBAL_UPDATE = 4,
-	S_CHECKPOINT = 5,
-
-	A_EXIT = 1,
-	A_CONTINUE = 2,
-	A_NEW_JOB = 3,
-	A_CONTINUE_JOB = 4,
-	A_PROCESS_DATA_NEW_JOB = 5,
-	A_GLOBAL_UPDATE	= 6,
-	A_CHECKPOINT = 7
-};

dump.h

@@ -119,7 +119,8 @@ constexpr hid_t iodump::h5_datatype() {
 
 	throw std::runtime_error{fmt::format("unsupported datatype: {}",typeid(T).name())};
 	// If you run into this error, you probably tried to write a non-primitive datatype
-	// to a dump file. see runner_task for a minimalistic example of what to do.
+	// to a dump file. See the other classes’s checkpointing functions for an example of
+	// what to do.
 	// ... or it is a native datatype I forgot to add. Then add it.
 }
 

evalable.cpp

 #include "evalable.h"
-#include "merger.h"
+#include <map>
 #include <fmt/format.h>
 
 evalable::evalable(const std::string& name, std::vector<std::string> used_observables, func fun)
@@ -10,7 +10,7 @@ const std::string& evalable::name() const {
 	return name_;
 }
 
-void evalable::jackknife(const merger_results& res, observable_result& obs_res) const {
+void evalable::jackknife(const results& res, observable_result& obs_res) const {
 	std::vector<observable_result> observables;
 	observables.reserve(used_observables_.size());
 

evalable.h

@@ -3,9 +3,7 @@
 #include <vector>
 #include <functional>
 #include <string>
-
-class observable_result;
-class merger_results;
+#include "results.h"
 
 class evalable {
 public:
@@ -20,7 +18,7 @@ public:
 	
 	evalable(const std::string& name, std::vector<std::string> used_observables, func fun); 
 	const std::string& name() const;
-	void jackknife(const merger_results& res, observable_result &out) const;
+	void jackknife(const results& res, observable_result &out) const;
 
 private:
         const std::string name_;

load_leveller.h

@@ -3,38 +3,26 @@
 #include "merger.h"
 #include "runner.h"
 #include "runner_single.h"
+#include "mc.h"
 
 namespace load_leveller {
-
 	template <class mc_runner>
-	static int run_mc(std::function<abstract_mc * (const std::string&)> mccreator, int argc, char **argv) {
-		if(argc < 4) {
-			std::cerr << "Usage: " << argv[0] <<" jobfile walltime checkpointtime [h/m/s]\nThe last argument sets the time unit for walltime and checkpointtime. Default is seconds.\n";
+	static int run_mc(mc_factory mccreator, int argc, char **argv) {
+		if(argc < 2) {
+			std::cerr << fmt::format("{0} JOBFILE\n{0} single JOBFILE\n{0} merge JOBFILE\n\n Without further flags, the MPI scheduler is started. 'single' starts a single core scheduler (useful for debugging), 'merge' merges unmerged measurement results.\n", argv[0]);
 			return -1;
 		}
 
-		std::string jobfile = argv[1];
-		double walltime = atof(argv[2]);
-		double chktime = atof(argv[3]);
-		if(argc>4) {//default is seconds
-			if(argv[4][0] == 'h') {
-				walltime *= 3600;
-				chktime *= 3600;
-			}
-			if(argv[4][0] == 'm') {
-				walltime *= 60;
-				chktime *= 60;
-			}
-		}
+		std::string jobfile{argv[1]};
 
 		mc_runner r;
-		return r.start(jobfile, walltime, chktime, mccreator, argc, argv);
+		return r.start(jobfile, mccreator, argc, argv);
 	}
 
 	// run this function from main() in your code.
 	template <class mc_implementation>
 	int run(int argc, char **argv) {
-		auto mccreator = [&] (const std::string& taskfile) -> abstract_mc* {return new mc_implementation(taskfile);};
+		auto mccreator = [&] (const std::string& jobfile, const std::string& taskname) -> abstract_mc* {return new mc_implementation(jobfile, taskname);};
 
 		if(argc > 1 && std::string(argv[1]) == "merge") {
 		//	return merge(mccreator, argc-1, argv+1);

mc.cpp

 #include "mc.h"
 
-abstract_mc::abstract_mc(const std::string& taskfile) {
-	param.read_file(taskfile);
-
-	therm_=param.value_or_default<int>("THERMALIZATION",10000);
+abstract_mc::abstract_mc(const std::string& jobfile, const std::string& taskname) 
+	: param{parser{jobfile}["tasks"][taskname]} {
+	
+	therm_ = param.get<int>("thermalization");
 }
 
 abstract_mc::~abstract_mc() {
 }
 
 void abstract_mc::random_init() {
-	if (param.defined("SEED"))
-		rng.reset(new randomnumbergenerator(param.value_of<uint64_t>("SEED")));
+	if(param.defined("seed"))
+		rng.reset(new randomnumbergenerator(param.get<uint64_t>("seed")));
 	else
 		rng.reset(new randomnumbergenerator());
 }

mc.h

@@ -3,10 +3,12 @@
 #include <memory>
 #include <vector>
 #include <string>
+#include <functional>
 #include "measurements.h"
 #include "random.h"
 #include "parser.h"
 
+
 class abstract_mc {
 private:
         void random_init();
@@ -41,6 +43,8 @@ public:
 	bool is_thermalized();
 	measurements measure;	
 	
-	abstract_mc(const std::string& taskfile);
+	abstract_mc(const std::string& jobfile, const std::string& taskname);
 	virtual ~abstract_mc();
 };
+
+typedef std::function<abstract_mc *(const std::string&, const std::string&)> mc_factory;

merger.cpp

@@ -18,8 +18,8 @@ void merger::add_evalable(const std::string& name, const std::vector<std::string
 	evalables_.emplace_back(name, used_observables, func);
 }
 
-merger_results merger::merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count) {
-	merger_results res;
+results merger::merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count) {
+	results res;
 
 	// This thing reads the complete time series of an observable which will
 	// probably make it the biggest memory user of load leveller. But since
@@ -159,7 +159,7 @@ merger_results merger::merge(const std::vector<std::string>& filenames, size_t r
 	return res;
 }
 
-void merger::evaluate_evalables(merger_results& res) {
+void merger::evaluate_evalables(results& res) {
 	std::vector<observable_result> evalable_results;
 	for(auto &eval : evalables_) {
 		evalable_results.emplace_back();

merger.h

@@ -4,43 +4,17 @@
 #include <map>
 
 #include "evalable.h"
-
-struct observable_result {
-	std::string name;
-
-	size_t rebinning_bin_length = 0;
-	size_t rebinning_bin_count = 0;
-	std::vector<double> rebinning_means; // [bin * vector_length + vector_idx]
-
-	size_t total_sample_count = 0;
-
-	// This is the bin length that was used when measuring the
-	// samples. If different runs had different internal_bin_lengths,
-	// this value is undefined.
-	//
-	// But it’s just a convenience feature, so that’s not bad.
-	size_t internal_bin_length = 0;
-
-	std::vector<double> mean;
-	std::vector<double> error;
-
-	std::vector<double> autocorrelation_time;
-};
-
-struct merger_results {
-	// contains both evalables and pure observables 
-	std::map<std::string, observable_result> observables;
-};
+#include "results.h"
 
 class merger {
 public:	
-	merger_results merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count = 32);
+	results merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count = 32);
 
 	// call this before merge
 	void add_evalable(const std::string& name, const std::vector<std::string>& used_observables, evalable::func func);
 private:
 	std::vector<evalable> evalables_;
-	void evaluate_evalables(merger_results &res);
+	void evaluate_evalables(results &res);
 };
 
 /*

parser.cpp

 #include "parser.h"
-#include <cstdlib>
-#include <fstream>
-#include <iostream>
-
-parser :: parser()
-{
-	notfine = " =\t";
-	comment = "#";	
-	array = "@";
-}
 
-parser :: parser(std::string input)
-{
-	notfine = " =\t";
-	comment = "#";
-	array = "@";
-	read_file(input);
+parser::iterator::iterator(std::string filename, YAML::Node::iterator it)
+	: filename_{std::move(filename)}, it_{std::move(it)} {
 }
 
-bool parser :: read_file(std::string input)
-{
-	std::string  buffer;
-	std::fstream file(input.c_str(), std::ios::in);
-	if (!file.good()) {
-		std::cerr << "File not found: " << input << std::endl; 
-		exit(1);
-		return false;
-	}
-	while(getline(file,buffer))
-	{
-		std::string::size_type where_is_comment = buffer.find_first_of(comment);
-		if(where_is_comment != buffer.npos)
-		{
-			buffer.erase(where_is_comment, buffer.size());
-		}
-		
-		std::string::size_type is_array = buffer.find_first_of(array);
-		if(is_array != buffer.npos)
-		{	
-			// we have to read an array
-			//std::string::size_type start = buffer.find_first_not_of(array); not used?
-			std::string::size_type end = buffer.find_first_of(array);
-			buffer.erase(0, end);
-			std::vector< std::string > dumpy;
-			std::string line2;
-			while(getline(file, line2) && line2.find(array) == line2.npos)
-			{
-				dumpy.push_back(line2);
-			}
-			arrs[buffer]=dumpy;
-		} else
-		{	// normal variable
-			std::string _name, value;
-			std::string::size_type start = buffer.find_first_not_of(notfine);
-			std::string::size_type end = buffer.find_first_of(notfine);
-			if(end != buffer.npos)
-			{
-				_name.assign(buffer, start, end);
-				buffer.erase(start, end);
-				end = buffer.find_first_not_of(notfine);
-				buffer.erase(0, end);
-				vars[_name]=buffer;
-			}
-		}
+std::pair<std::string, parser> parser::iterator::operator*() {
+	try {
+		return std::make_pair(it_->first.as<std::string>(), parser{it_->second, filename_});
+	} catch(YAML::Exception e) {
+		throw std::runtime_error(fmt::format("YAML: {}: dereferencing map key failed: {}. Maybe it was not a string?", filename_, e.what()));
 	}
-	file.close();
-	readVals.clear();
-	return true;
 }
 
-std::string parser::return_value_of(std::string name)
-{
-	std::map<std::string,std::string>::iterator it;
-	it=vars.find(name);
-	if(it!=vars.end())
-	{
-		readVals.push_back(name);
-		return (*it).second;
-	}
-	else {
-		std::cerr << "#PARSER: NO VARIABLE WITH NAME " << name << " FOUND!" << std::endl;
-		return "";
-	}
+static std::runtime_error non_map_error(const std::string& filename) {
+	return std::runtime_error(fmt::format("YAML: {}: trying to dereference non-map node.", filename));
 }
 
-std::string parser::value_or_default(std::string name, std::string defa)
-{
-	std::map<std::string,std::string>::iterator it;
-	it=vars.find(name);
-	if(it!=vars.end()) {
-		readVals.push_back(name);
-		return (*it).second;
-	}
-	else return defa;
+static std::runtime_error key_error(const std::string& filename, const std::string& name) {
+	return std::runtime_error(fmt::format("YAML: {}: could not find required key '{}'", filename, name));
 }
 
-bool parser::defined(std::string name)
-{
-	std::map<std::string,std::string>::iterator it;
-	it=vars.find(name);
-	std::map<std::string,std::vector<std::string> >::iterator arrit = arrs.find(name);
-	return it != vars.end() || arrit != arrs.end();
-//	if(it==vars.end()) return false;
-//	else return true;
-}	
-
-void parser :: add_comment(std::string new_comm)
-{
-	comment += new_comm;
+
+parser::iterator parser::iterator::operator++() {
+	return iterator{filename_, it_++};
 }
 
-void parser :: add_delim(std::string new_del)
-{
-	notfine += new_del;
+bool parser::iterator::operator!=(const iterator& b) {
+	return it_ != b.it_;
 }
 
-void parser :: get_all(std::ostream& os)
-{
-	std::map<std::string,std::string>::iterator i;
-	std::map<std::string,std::vector<std::string> >::iterator i2;
-	for(i=vars.begin(); i!=vars.end(); i++)
-		os<<(*i).first<<" = "<<(*i).second<< std::endl;
-
-	for(i2=arrs.begin(); i2!=arrs.end();i2++) {
-		os<<(*i2).first<<"\n";
-		int size=((*i2).second).size();
-		for(int j=0;j<size;j++)
-			os<< ((*i2).second)[j] <<"\n";
-		os<<(*i2).first<< std::endl;
-	}
-}	
-
-void parser :: get_all_with_one_from_specified_array(std::string sfai, int fai, std::ostream& os)
-{
-	std::map<std::string,std::string>::iterator i;
-	std::map<std::string,std::vector<std::string> >::iterator i2;
-	
-	for(i=vars.begin(); i!=vars.end(); i++)
-		os<<(*i).first<<" = "<<(*i).second<<"\n";
-
-	for(i2=arrs.begin(); i2!=arrs.end();i2++) {
-		if((*i2).first==sfai)
-			os << (*i2).first.substr(1) << " = " << ((*i2).second)[fai] << "\n";
-		else {
-			os<<(*i2).first<<"\n";
-			int size=((*i2).second).size();
-			for(int j=0;j<size;j++)
-				os<< ((*i2).second)[j] <<"\n";
-			os<<(*i2).first<<"\n";
-
-		}
+parser::parser(const YAML::Node& node, const std::string& filename) 
+	: content_{node}, filename_{filename} {
+	if(!content_.IsMap()) {
+		throw non_map_error(filename);
 	}
 }
 
-std::vector<std::string> parser::unused_parameters() {
-	std::vector<std::string> unused;
-	std::vector<std::string>::iterator re;
-	bool in;
-	for (std::map<std::string, std::string>::iterator va = vars.begin(); va != vars.end(); ++va) {
-		in = false;
-		for (re = readVals.begin(); re != readVals.end(); ++re) {
-			if (va->first == (*re)) {
-				in = true;
-				break;
-			}
-		}
-		if (!in)
-			unused.push_back(va->first);
+parser::parser(const std::string& filename)
+	: parser{YAML::LoadFile(filename), filename} {
+}
+
+parser::iterator parser::begin() {
+	if(!content_.IsMap()) {
+		throw non_map_error(filename_);
 	}
-	for (std::map<std::string, std::vector<std::string> >::iterator va = arrs.begin(); va != arrs.end(); ++va) {
-		in = false;
-		for (re = readVals.begin(); re != readVals.end(); ++re) {
-			if (va->first == (*re)) {
-				in = true;
-				break;
-			}
-		}
-		if (!in)
-			unused.push_back(va->first);
+
+	return iterator{filename_, content_.begin()};
+}
+
+parser::iterator parser::end() {
+
+	return iterator{filename_, content_.begin()};
+}
+
+bool parser::defined(const std::string& name) {
+	if(!content_.IsMap()) {
+		return false;
 	}
-	return unused;
+	return content_[name].IsDefined();
 }
 
+parser parser::operator[](const std::string& name) {
+	if(!content_.IsMap()) {
+		throw non_map_error(filename_);
+	}
+		
+	auto node = content_[name];
+	if(!node.IsDefined())
+		throw key_error(filename_, name);
+	if(!node.IsMap())
+		throw std::runtime_error(fmt::format("YAML: {}: Found key '{}', but it has a scalar value. Was expecting it to be a map", filename_, name));
+
+	try {
+		return parser{node, filename_};
+	} catch(YAML::Exception) {
+		throw key_error(filename_, name);
+	}
+}

parser.h

-#ifndef MY_PARSER_H
-#define MY_PARSER_H
-
-#include <iostream>
-#include <vector>
-#include <string>
-#include <sstream>
-#include <map>
-
-class parser
-{
+#pragma once
+
+#include <yaml-cpp/yaml.h>
+#include <fmt/format.h>
+
+// This is mostly a wrapper around yaml-cpp with more helpful error handling.
+// For simplicity it does not support advanced yaml features such as complex-typed
+// keys in maps.
+
+class parser {
+private:
+	YAML::Node content_;
+	const std::string filename_;
+	
+	// fake parser based on a subnode
+	parser(const YAML::Node& node, const std::string& filename);
+public:
+	class iterator {
+	private:
+		std::string filename_;
+		YAML::Node::iterator it_;
 	public:
-		parser();
-		parser(std::string);
-		
-		// one can add other symbols for comments, etc.
-		void add_comment(std::string);
-		void add_delim(std::string);
-		
-		// read a file (if it has not happend yet)
-		bool read_file(std::string);
-		
-		// writes out all variables to the specified output
-		void get_all(std::ostream&);
-
-		// writes out all scalar variables and from the first array 
-		// the secified entry to the specified output
-		// usefull for output within a parallel tempering code
-		void get_all_with_one_from_specified_array(std::string, int, std::ostream&);
-
-		std::string return_value_of(std::string);
-		std::string value_of(std::string name) {return return_value_of(name);}
-		std::string operator[](std::string name) {return return_value_of(name);}
-
-		bool defined(std::string);
-
-		template <class Type>
-		Type return_value_of(std::string name)
-		{
-			Type ret;
-			std::stringstream buffer;
-			std::map<std::string,std::string>::iterator it;
-			it=vars.find(name);
-			if(it != vars.end()) {	
-				buffer << vars[name];
-				buffer >> ret;
-				readVals.push_back(name);
-				return ret;
-			} else {
-				std::cerr<<"#PARSER: NO VARIABLE WITH NAME "<<name<<" FOUND!"<< std::endl;
-				return Type();
-			}
-		}
-		
-		template <class Type>
-		Type value_of(std::string name) {return return_value_of<Type>(name);}
-
-		template <class Type>
-		Type operator[](std::string name) {return return_value_of<Type>(name);}
-
-		std::string value_or_default(std::string, std::string);
-		
-		template <class Type>
-		Type value_or_default(std::string name, Type defa)
-		{
-			Type ret;
-			std::stringstream buffer;
-			std::map<std::string,std::string>::iterator it;
-			it=vars.find(name);
-			if(it != vars.end()) {	
-				buffer << vars[name];
-				buffer >> ret;
-				readVals.push_back(name);
-				return ret;
-			} else return defa;
-		}
-
-		// shorter wrapper for value_or_default
-		// be very careful when using it because the return type is inferred from the type of defa. 0. and 0 are different!
-		template <class T>
-		auto get(const std::string &parameter, T defa) -> decltype(defa) {
-			return value_or_default(parameter, defa);
-		}
-
-
-		template <class Type>
-		void set_value(std::string name, Type value)
-		{
-			std::stringstream b;
-			b<<value;
-			vars[name]=b.str();
-		}
-
-		template <class Type>
-		std::vector< Type > return_vector(std::string name)
-		{
-			std::vector< Type > ret;
-			std::map<std::string,std::vector<std::string> >::iterator it;
-			it=arrs.find(name);
-			if(it==arrs.end()) {
-				std::cerr<<"#PARSER: NO ARRAY WITH NAME "<<name<<" FOUND!"<<std::endl; 
-				return ret;
-			}
-			for(uint j=0;j<((*it).second).size();j++) {
-				Type tmp;
-				std::stringstream buffer;