Commit 2eb293be authored by Lukas Weber's avatar Lukas Weber
Browse files

yaml parser replacement

parent 136d4e38
#pragma once
enum {
MASTER = 0,
T_STATUS = 1,
T_ACTION = 2,
T_NEW_JOB = 3,
T_PARTNER = 4,
T_WEIGHT = 5,
T_LABEL = 6,
T_POS = 7,
S_IDLE = 1,
S_BUSY = 2,
S_FINISHED = 3,
S_GLOBAL_UPDATE = 4,
S_CHECKPOINT = 5,
A_EXIT = 1,
A_CONTINUE = 2,
A_NEW_JOB = 3,
A_CONTINUE_JOB = 4,
A_PROCESS_DATA_NEW_JOB = 5,
A_GLOBAL_UPDATE = 6,
A_CHECKPOINT = 7
};
......@@ -119,7 +119,8 @@ constexpr hid_t iodump::h5_datatype() {
throw std::runtime_error{fmt::format("unsupported datatype: {}",typeid(T).name())};
// If you run into this error, you probably tried to write a non-primitive datatype
// to a dump file. see runner_task for a minimalistic example of what to do.
// to a dump file. See the other classes’s checkpointing functions for an example of
// what to do.
// ... or it is a native datatype I forgot to add. Then add it.
}
......
#include "evalable.h"
#include "merger.h"
#include <map>
#include <fmt/format.h>
evalable::evalable(const std::string& name, std::vector<std::string> used_observables, func fun)
......@@ -10,7 +10,7 @@ const std::string& evalable::name() const {
return name_;
}
void evalable::jackknife(const merger_results& res, observable_result& obs_res) const {
void evalable::jackknife(const results& res, observable_result& obs_res) const {
std::vector<observable_result> observables;
observables.reserve(used_observables_.size());
......
......@@ -3,9 +3,7 @@
#include <vector>
#include <functional>
#include <string>
class observable_result;
class merger_results;
#include "results.h"
class evalable {
public:
......@@ -20,7 +18,7 @@ public:
evalable(const std::string& name, std::vector<std::string> used_observables, func fun);
const std::string& name() const;
void jackknife(const merger_results& res, observable_result &out) const;
void jackknife(const results& res, observable_result &out) const;
private:
const std::string name_;
......
......@@ -3,38 +3,26 @@
#include "merger.h"
#include "runner.h"
#include "runner_single.h"
#include "mc.h"
namespace load_leveller {
template <class mc_runner>
static int run_mc(std::function<abstract_mc * (const std::string&)> mccreator, int argc, char **argv) {
if(argc < 4) {
std::cerr << "Usage: " << argv[0] <<" jobfile walltime checkpointtime [h/m/s]\nThe last argument sets the time unit for walltime and checkpointtime. Default is seconds.\n";
static int run_mc(mc_factory mccreator, int argc, char **argv) {
if(argc < 2) {
std::cerr << fmt::format("{0} JOBFILE\n{0} single JOBFILE\n{0} merge JOBFILE\n\n Without further flags, the MPI scheduler is started. 'single' starts a single core scheduler (useful for debugging), 'merge' merges unmerged measurement results.\n", argv[0]);
return -1;
}
std::string jobfile = argv[1];
double walltime = atof(argv[2]);
double chktime = atof(argv[3]);
if(argc>4) {//default is seconds
if(argv[4][0] == 'h') {
walltime *= 3600;
chktime *= 3600;
}
if(argv[4][0] == 'm') {
walltime *= 60;
chktime *= 60;
}
}
std::string jobfile{argv[1]};
mc_runner r;
return r.start(jobfile, walltime, chktime, mccreator, argc, argv);
return r.start(jobfile, mccreator, argc, argv);
}
// run this function from main() in your code.
template <class mc_implementation>
int run(int argc, char **argv) {
auto mccreator = [&] (const std::string& taskfile) -> abstract_mc* {return new mc_implementation(taskfile);};
auto mccreator = [&] (const std::string& jobfile, const std::string& taskname) -> abstract_mc* {return new mc_implementation(jobfile, taskname);};
if(argc > 1 && std::string(argv[1]) == "merge") {
// return merge(mccreator, argc-1, argv+1);
......
#include "mc.h"
abstract_mc::abstract_mc(const std::string& taskfile) {
param.read_file(taskfile);
therm_=param.value_or_default<int>("THERMALIZATION",10000);
abstract_mc::abstract_mc(const std::string& jobfile, const std::string& taskname)
: param{parser{jobfile}["tasks"][taskname]} {
therm_ = param.get<int>("thermalization");
}
abstract_mc::~abstract_mc() {
}
void abstract_mc::random_init() {
if (param.defined("SEED"))
rng.reset(new randomnumbergenerator(param.value_of<uint64_t>("SEED")));
if(param.defined("seed"))
rng.reset(new randomnumbergenerator(param.get<uint64_t>("seed")));
else
rng.reset(new randomnumbergenerator());
}
......
......@@ -3,10 +3,12 @@
#include <memory>
#include <vector>
#include <string>
#include <functional>
#include "measurements.h"
#include "random.h"
#include "parser.h"
class abstract_mc {
private:
void random_init();
......@@ -41,6 +43,8 @@ public:
bool is_thermalized();
measurements measure;
abstract_mc(const std::string& taskfile);
abstract_mc(const std::string& jobfile, const std::string& taskname);
virtual ~abstract_mc();
};
typedef std::function<abstract_mc *(const std::string&, const std::string&)> mc_factory;
......@@ -18,8 +18,8 @@ void merger::add_evalable(const std::string& name, const std::vector<std::string
evalables_.emplace_back(name, used_observables, func);
}
merger_results merger::merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count) {
merger_results res;
results merger::merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count) {
results res;
// This thing reads the complete time series of an observable which will
// probably make it the biggest memory user of load leveller. But since
......@@ -159,7 +159,7 @@ merger_results merger::merge(const std::vector<std::string>& filenames, size_t r
return res;
}
void merger::evaluate_evalables(merger_results& res) {
void merger::evaluate_evalables(results& res) {
std::vector<observable_result> evalable_results;
for(auto &eval : evalables_) {
evalable_results.emplace_back();
......
......@@ -4,43 +4,17 @@
#include <map>
#include "evalable.h"
struct observable_result {
std::string name;
size_t rebinning_bin_length = 0;
size_t rebinning_bin_count = 0;
std::vector<double> rebinning_means; // [bin * vector_length + vector_idx]
size_t total_sample_count = 0;
// This is the bin length that was used when measuring the
// samples. If different runs had different internal_bin_lengths,
// this value is undefined.
//
// But it’s just a convenience feature, so that’s not bad.
size_t internal_bin_length = 0;
std::vector<double> mean;
std::vector<double> error;
std::vector<double> autocorrelation_time;
};
struct merger_results {
// contains both evalables and pure observables
std::map<std::string, observable_result> observables;
};
#include "results.h"
class merger {
public:
merger_results merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count = 32);
results merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count = 32);
// call this before merge
void add_evalable(const std::string& name, const std::vector<std::string>& used_observables, evalable::func func);
private:
std::vector<evalable> evalables_;
void evaluate_evalables(merger_results &res);
void evaluate_evalables(results &res);
};
/*
......
#include "parser.h"
#include <cstdlib>
#include <fstream>
#include <iostream>
parser :: parser()
{
notfine = " =\t";
comment = "#";
array = "@";
}
parser :: parser(std::string input)
{
notfine = " =\t";
comment = "#";
array = "@";
read_file(input);
parser::iterator::iterator(std::string filename, YAML::Node::iterator it)
: filename_{std::move(filename)}, it_{std::move(it)} {
}
bool parser :: read_file(std::string input)
{
std::string buffer;
std::fstream file(input.c_str(), std::ios::in);
if (!file.good()) {
std::cerr << "File not found: " << input << std::endl;
exit(1);
return false;
}
while(getline(file,buffer))
{
std::string::size_type where_is_comment = buffer.find_first_of(comment);
if(where_is_comment != buffer.npos)
{
buffer.erase(where_is_comment, buffer.size());
}
std::string::size_type is_array = buffer.find_first_of(array);
if(is_array != buffer.npos)
{
// we have to read an array
//std::string::size_type start = buffer.find_first_not_of(array); not used?
std::string::size_type end = buffer.find_first_of(array);
buffer.erase(0, end);
std::vector< std::string > dumpy;
std::string line2;
while(getline(file, line2) && line2.find(array) == line2.npos)
{
dumpy.push_back(line2);
}
arrs[buffer]=dumpy;
} else
{ // normal variable
std::string _name, value;
std::string::size_type start = buffer.find_first_not_of(notfine);
std::string::size_type end = buffer.find_first_of(notfine);
if(end != buffer.npos)
{
_name.assign(buffer, start, end);
buffer.erase(start, end);
end = buffer.find_first_not_of(notfine);
buffer.erase(0, end);
vars[_name]=buffer;
}
}
std::pair<std::string, parser> parser::iterator::operator*() {
try {
return std::make_pair(it_->first.as<std::string>(), parser{it_->second, filename_});
} catch(YAML::Exception e) {
throw std::runtime_error(fmt::format("YAML: {}: dereferencing map key failed: {}. Maybe it was not a string?", filename_, e.what()));
}
file.close();
readVals.clear();
return true;
}
std::string parser::return_value_of(std::string name)
{
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
if(it!=vars.end())
{
readVals.push_back(name);
return (*it).second;
}
else {
std::cerr << "#PARSER: NO VARIABLE WITH NAME " << name << " FOUND!" << std::endl;
return "";
}
static std::runtime_error non_map_error(const std::string& filename) {
return std::runtime_error(fmt::format("YAML: {}: trying to dereference non-map node.", filename));
}
std::string parser::value_or_default(std::string name, std::string defa)
{
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
if(it!=vars.end()) {
readVals.push_back(name);
return (*it).second;
}
else return defa;
static std::runtime_error key_error(const std::string& filename, const std::string& name) {
return std::runtime_error(fmt::format("YAML: {}: could not find required key '{}'", filename, name));
}
bool parser::defined(std::string name)
{
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
std::map<std::string,std::vector<std::string> >::iterator arrit = arrs.find(name);
return it != vars.end() || arrit != arrs.end();
// if(it==vars.end()) return false;
// else return true;
}
void parser :: add_comment(std::string new_comm)
{
comment += new_comm;
parser::iterator parser::iterator::operator++() {
return iterator{filename_, it_++};
}
void parser :: add_delim(std::string new_del)
{
notfine += new_del;
bool parser::iterator::operator!=(const iterator& b) {
return it_ != b.it_;
}
void parser :: get_all(std::ostream& os)
{
std::map<std::string,std::string>::iterator i;
std::map<std::string,std::vector<std::string> >::iterator i2;
for(i=vars.begin(); i!=vars.end(); i++)
os<<(*i).first<<" = "<<(*i).second<< std::endl;
for(i2=arrs.begin(); i2!=arrs.end();i2++) {
os<<(*i2).first<<"\n";
int size=((*i2).second).size();
for(int j=0;j<size;j++)
os<< ((*i2).second)[j] <<"\n";
os<<(*i2).first<< std::endl;
}
}
void parser :: get_all_with_one_from_specified_array(std::string sfai, int fai, std::ostream& os)
{
std::map<std::string,std::string>::iterator i;
std::map<std::string,std::vector<std::string> >::iterator i2;
for(i=vars.begin(); i!=vars.end(); i++)
os<<(*i).first<<" = "<<(*i).second<<"\n";
for(i2=arrs.begin(); i2!=arrs.end();i2++) {
if((*i2).first==sfai)
os << (*i2).first.substr(1) << " = " << ((*i2).second)[fai] << "\n";
else {
os<<(*i2).first<<"\n";
int size=((*i2).second).size();
for(int j=0;j<size;j++)
os<< ((*i2).second)[j] <<"\n";
os<<(*i2).first<<"\n";
}
parser::parser(const YAML::Node& node, const std::string& filename)
: content_{node}, filename_{filename} {
if(!content_.IsMap()) {
throw non_map_error(filename);
}
}
std::vector<std::string> parser::unused_parameters() {
std::vector<std::string> unused;
std::vector<std::string>::iterator re;
bool in;
for (std::map<std::string, std::string>::iterator va = vars.begin(); va != vars.end(); ++va) {
in = false;
for (re = readVals.begin(); re != readVals.end(); ++re) {
if (va->first == (*re)) {
in = true;
break;
}
}
if (!in)
unused.push_back(va->first);
parser::parser(const std::string& filename)
: parser{YAML::LoadFile(filename), filename} {
}
parser::iterator parser::begin() {
if(!content_.IsMap()) {
throw non_map_error(filename_);
}
for (std::map<std::string, std::vector<std::string> >::iterator va = arrs.begin(); va != arrs.end(); ++va) {
in = false;
for (re = readVals.begin(); re != readVals.end(); ++re) {
if (va->first == (*re)) {
in = true;
break;
}
}
if (!in)
unused.push_back(va->first);
return iterator{filename_, content_.begin()};
}
parser::iterator parser::end() {
return iterator{filename_, content_.begin()};
}
bool parser::defined(const std::string& name) {
if(!content_.IsMap()) {
return false;
}
return unused;
return content_[name].IsDefined();
}
parser parser::operator[](const std::string& name) {
if(!content_.IsMap()) {
throw non_map_error(filename_);
}
auto node = content_[name];
if(!node.IsDefined())
throw key_error(filename_, name);
if(!node.IsMap())
throw std::runtime_error(fmt::format("YAML: {}: Found key '{}', but it has a scalar value. Was expecting it to be a map", filename_, name));
try {
return parser{node, filename_};
} catch(YAML::Exception) {
throw key_error(filename_, name);
}
}
#ifndef MY_PARSER_H
#define MY_PARSER_H
#include <iostream>
#include <vector>
#include <string>
#include <sstream>
#include <map>
class parser
{
#pragma once
#include <yaml-cpp/yaml.h>
#include <fmt/format.h>
// This is mostly a wrapper around yaml-cpp with more helpful error handling.
// For simplicity it does not support advanced yaml features such as complex-typed
// keys in maps.
class parser {
private:
YAML::Node content_;
const std::string filename_;
// fake parser based on a subnode
parser(const YAML::Node& node, const std::string& filename);
public:
class iterator {
private:
std::string filename_;
YAML::Node::iterator it_;
public:
parser();
parser(std::string);
// one can add other symbols for comments, etc.
void add_comment(std::string);
void add_delim(std::string);
// read a file (if it has not happend yet)
bool read_file(std::string);
// writes out all variables to the specified output
void get_all(std::ostream&);
// writes out all scalar variables and from the first array
// the secified entry to the specified output
// usefull for output within a parallel tempering code
void get_all_with_one_from_specified_array(std::string, int, std::ostream&);
std::string return_value_of(std::string);
std::string value_of(std::string name) {return return_value_of(name);}
std::string operator[](std::string name) {return return_value_of(name);}
bool defined(std::string);
template <class Type>
Type return_value_of(std::string name)
{
Type ret;
std::stringstream buffer;
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
if(it != vars.end()) {
buffer << vars[name];
buffer >> ret;
readVals.push_back(name);
return ret;
} else {
std::cerr<<"#PARSER: NO VARIABLE WITH NAME "<<name<<" FOUND!"<< std::endl;
return Type();
}
}
template <class Type>
Type value_of(std::string name) {return return_value_of<Type>(name);}
template <class Type>
Type operator[](std::string name) {return return_value_of<Type>(name);}
std::string value_or_default(std::string, std::string);
template <class Type>
Type value_or_default(std::string name, Type defa)
{
Type ret;
std::stringstream buffer;
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
if(it != vars.end()) {
buffer << vars[name];
buffer >> ret;
readVals.push_back(name);
return ret;
} else return defa;
}
// shorter wrapper for value_or_default
// be very careful when using it because the return type is inferred from the type of defa. 0. and 0 are different!
template <class T>
auto get(const std::string &parameter, T defa) -> decltype(defa) {
return value_or_default(parameter, defa);
}
template <class Type>
void set_value(std::string name, Type value)
{
std::stringstream b;
b<<value;
vars[name]=b.str();
}
template <class Type>
std::vector< Type > return_vector(std::string name)
{
std::vector< Type > ret;
std::map<std::string,std::vector<std::string> >::iterator it;
it=arrs.find(name);
if(it==arrs.end()) {
std::cerr<<"#PARSER: NO ARRAY WITH NAME "<<name<<" FOUND!"<<std::endl;
return ret;
}
for(uint j=0;j<((*it).second).size();j++) {
Type tmp;
std::stringstream buffer;