Commit 2eb293be authored by Lukas Weber's avatar Lukas Weber
Browse files

yaml parser replacement

parent 136d4e38
#pragma once
enum {
MASTER = 0,
T_STATUS = 1,
T_ACTION = 2,
T_NEW_JOB = 3,
T_PARTNER = 4,
T_WEIGHT = 5,
T_LABEL = 6,
T_POS = 7,
S_IDLE = 1,
S_BUSY = 2,
S_FINISHED = 3,
S_GLOBAL_UPDATE = 4,
S_CHECKPOINT = 5,
A_EXIT = 1,
A_CONTINUE = 2,
A_NEW_JOB = 3,
A_CONTINUE_JOB = 4,
A_PROCESS_DATA_NEW_JOB = 5,
A_GLOBAL_UPDATE = 6,
A_CHECKPOINT = 7
};
...@@ -119,7 +119,8 @@ constexpr hid_t iodump::h5_datatype() { ...@@ -119,7 +119,8 @@ constexpr hid_t iodump::h5_datatype() {
throw std::runtime_error{fmt::format("unsupported datatype: {}",typeid(T).name())}; throw std::runtime_error{fmt::format("unsupported datatype: {}",typeid(T).name())};
// If you run into this error, you probably tried to write a non-primitive datatype // If you run into this error, you probably tried to write a non-primitive datatype
// to a dump file. see runner_task for a minimalistic example of what to do. // to a dump file. See the other classes’s checkpointing functions for an example of
// what to do.
// ... or it is a native datatype I forgot to add. Then add it. // ... or it is a native datatype I forgot to add. Then add it.
} }
......
#include "evalable.h" #include "evalable.h"
#include "merger.h" #include <map>
#include <fmt/format.h> #include <fmt/format.h>
evalable::evalable(const std::string& name, std::vector<std::string> used_observables, func fun) evalable::evalable(const std::string& name, std::vector<std::string> used_observables, func fun)
...@@ -10,7 +10,7 @@ const std::string& evalable::name() const { ...@@ -10,7 +10,7 @@ const std::string& evalable::name() const {
return name_; return name_;
} }
void evalable::jackknife(const merger_results& res, observable_result& obs_res) const { void evalable::jackknife(const results& res, observable_result& obs_res) const {
std::vector<observable_result> observables; std::vector<observable_result> observables;
observables.reserve(used_observables_.size()); observables.reserve(used_observables_.size());
......
...@@ -3,9 +3,7 @@ ...@@ -3,9 +3,7 @@
#include <vector> #include <vector>
#include <functional> #include <functional>
#include <string> #include <string>
#include "results.h"
class observable_result;
class merger_results;
class evalable { class evalable {
public: public:
...@@ -20,7 +18,7 @@ public: ...@@ -20,7 +18,7 @@ public:
evalable(const std::string& name, std::vector<std::string> used_observables, func fun); evalable(const std::string& name, std::vector<std::string> used_observables, func fun);
const std::string& name() const; const std::string& name() const;
void jackknife(const merger_results& res, observable_result &out) const; void jackknife(const results& res, observable_result &out) const;
private: private:
const std::string name_; const std::string name_;
......
...@@ -3,38 +3,26 @@ ...@@ -3,38 +3,26 @@
#include "merger.h" #include "merger.h"
#include "runner.h" #include "runner.h"
#include "runner_single.h" #include "runner_single.h"
#include "mc.h"
namespace load_leveller { namespace load_leveller {
template <class mc_runner> template <class mc_runner>
static int run_mc(std::function<abstract_mc * (const std::string&)> mccreator, int argc, char **argv) { static int run_mc(mc_factory mccreator, int argc, char **argv) {
if(argc < 4) { if(argc < 2) {
std::cerr << "Usage: " << argv[0] <<" jobfile walltime checkpointtime [h/m/s]\nThe last argument sets the time unit for walltime and checkpointtime. Default is seconds.\n"; std::cerr << fmt::format("{0} JOBFILE\n{0} single JOBFILE\n{0} merge JOBFILE\n\n Without further flags, the MPI scheduler is started. 'single' starts a single core scheduler (useful for debugging), 'merge' merges unmerged measurement results.\n", argv[0]);
return -1; return -1;
} }
std::string jobfile = argv[1]; std::string jobfile{argv[1]};
double walltime = atof(argv[2]);
double chktime = atof(argv[3]);
if(argc>4) {//default is seconds
if(argv[4][0] == 'h') {
walltime *= 3600;
chktime *= 3600;
}
if(argv[4][0] == 'm') {
walltime *= 60;
chktime *= 60;
}
}
mc_runner r; mc_runner r;
return r.start(jobfile, walltime, chktime, mccreator, argc, argv); return r.start(jobfile, mccreator, argc, argv);
} }
// run this function from main() in your code. // run this function from main() in your code.
template <class mc_implementation> template <class mc_implementation>
int run(int argc, char **argv) { int run(int argc, char **argv) {
auto mccreator = [&] (const std::string& taskfile) -> abstract_mc* {return new mc_implementation(taskfile);}; auto mccreator = [&] (const std::string& jobfile, const std::string& taskname) -> abstract_mc* {return new mc_implementation(jobfile, taskname);};
if(argc > 1 && std::string(argv[1]) == "merge") { if(argc > 1 && std::string(argv[1]) == "merge") {
// return merge(mccreator, argc-1, argv+1); // return merge(mccreator, argc-1, argv+1);
......
#include "mc.h" #include "mc.h"
abstract_mc::abstract_mc(const std::string& taskfile) { abstract_mc::abstract_mc(const std::string& jobfile, const std::string& taskname)
param.read_file(taskfile); : param{parser{jobfile}["tasks"][taskname]} {
therm_=param.value_or_default<int>("THERMALIZATION",10000); therm_ = param.get<int>("thermalization");
} }
abstract_mc::~abstract_mc() { abstract_mc::~abstract_mc() {
} }
void abstract_mc::random_init() { void abstract_mc::random_init() {
if (param.defined("SEED")) if(param.defined("seed"))
rng.reset(new randomnumbergenerator(param.value_of<uint64_t>("SEED"))); rng.reset(new randomnumbergenerator(param.get<uint64_t>("seed")));
else else
rng.reset(new randomnumbergenerator()); rng.reset(new randomnumbergenerator());
} }
......
...@@ -3,10 +3,12 @@ ...@@ -3,10 +3,12 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include <string> #include <string>
#include <functional>
#include "measurements.h" #include "measurements.h"
#include "random.h" #include "random.h"
#include "parser.h" #include "parser.h"
class abstract_mc { class abstract_mc {
private: private:
void random_init(); void random_init();
...@@ -41,6 +43,8 @@ public: ...@@ -41,6 +43,8 @@ public:
bool is_thermalized(); bool is_thermalized();
measurements measure; measurements measure;
abstract_mc(const std::string& taskfile); abstract_mc(const std::string& jobfile, const std::string& taskname);
virtual ~abstract_mc(); virtual ~abstract_mc();
}; };
typedef std::function<abstract_mc *(const std::string&, const std::string&)> mc_factory;
...@@ -18,8 +18,8 @@ void merger::add_evalable(const std::string& name, const std::vector<std::string ...@@ -18,8 +18,8 @@ void merger::add_evalable(const std::string& name, const std::vector<std::string
evalables_.emplace_back(name, used_observables, func); evalables_.emplace_back(name, used_observables, func);
} }
merger_results merger::merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count) { results merger::merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count) {
merger_results res; results res;
// This thing reads the complete time series of an observable which will // This thing reads the complete time series of an observable which will
// probably make it the biggest memory user of load leveller. But since // probably make it the biggest memory user of load leveller. But since
...@@ -159,7 +159,7 @@ merger_results merger::merge(const std::vector<std::string>& filenames, size_t r ...@@ -159,7 +159,7 @@ merger_results merger::merge(const std::vector<std::string>& filenames, size_t r
return res; return res;
} }
void merger::evaluate_evalables(merger_results& res) { void merger::evaluate_evalables(results& res) {
std::vector<observable_result> evalable_results; std::vector<observable_result> evalable_results;
for(auto &eval : evalables_) { for(auto &eval : evalables_) {
evalable_results.emplace_back(); evalable_results.emplace_back();
......
...@@ -4,43 +4,17 @@ ...@@ -4,43 +4,17 @@
#include <map> #include <map>
#include "evalable.h" #include "evalable.h"
#include "results.h"
struct observable_result {
std::string name;
size_t rebinning_bin_length = 0;
size_t rebinning_bin_count = 0;
std::vector<double> rebinning_means; // [bin * vector_length + vector_idx]
size_t total_sample_count = 0;
// This is the bin length that was used when measuring the
// samples. If different runs had different internal_bin_lengths,
// this value is undefined.
//
// But it’s just a convenience feature, so that’s not bad.
size_t internal_bin_length = 0;
std::vector<double> mean;
std::vector<double> error;
std::vector<double> autocorrelation_time;
};
struct merger_results {
// contains both evalables and pure observables
std::map<std::string, observable_result> observables;
};
class merger { class merger {
public: public:
merger_results merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count = 32); results merge(const std::vector<std::string>& filenames, size_t rebinning_bin_count = 32);
// call this before merge // call this before merge
void add_evalable(const std::string& name, const std::vector<std::string>& used_observables, evalable::func func); void add_evalable(const std::string& name, const std::vector<std::string>& used_observables, evalable::func func);
private: private:
std::vector<evalable> evalables_; std::vector<evalable> evalables_;
void evaluate_evalables(merger_results &res); void evaluate_evalables(results &res);
}; };
/* /*
......
#include "parser.h" #include "parser.h"
#include <cstdlib>
#include <fstream>
#include <iostream>
parser :: parser()
{
notfine = " =\t";
comment = "#";
array = "@";
}
parser :: parser(std::string input) parser::iterator::iterator(std::string filename, YAML::Node::iterator it)
{ : filename_{std::move(filename)}, it_{std::move(it)} {
notfine = " =\t";
comment = "#";
array = "@";
read_file(input);
} }
bool parser :: read_file(std::string input) std::pair<std::string, parser> parser::iterator::operator*() {
{ try {
std::string buffer; return std::make_pair(it_->first.as<std::string>(), parser{it_->second, filename_});
std::fstream file(input.c_str(), std::ios::in); } catch(YAML::Exception e) {
if (!file.good()) { throw std::runtime_error(fmt::format("YAML: {}: dereferencing map key failed: {}. Maybe it was not a string?", filename_, e.what()));
std::cerr << "File not found: " << input << std::endl;
exit(1);
return false;
}
while(getline(file,buffer))
{
std::string::size_type where_is_comment = buffer.find_first_of(comment);
if(where_is_comment != buffer.npos)
{
buffer.erase(where_is_comment, buffer.size());
}
std::string::size_type is_array = buffer.find_first_of(array);
if(is_array != buffer.npos)
{
// we have to read an array
//std::string::size_type start = buffer.find_first_not_of(array); not used?
std::string::size_type end = buffer.find_first_of(array);
buffer.erase(0, end);
std::vector< std::string > dumpy;
std::string line2;
while(getline(file, line2) && line2.find(array) == line2.npos)
{
dumpy.push_back(line2);
}
arrs[buffer]=dumpy;
} else
{ // normal variable
std::string _name, value;
std::string::size_type start = buffer.find_first_not_of(notfine);
std::string::size_type end = buffer.find_first_of(notfine);
if(end != buffer.npos)
{
_name.assign(buffer, start, end);
buffer.erase(start, end);
end = buffer.find_first_not_of(notfine);
buffer.erase(0, end);
vars[_name]=buffer;
}
}
} }
file.close();
readVals.clear();
return true;
} }
std::string parser::return_value_of(std::string name) static std::runtime_error non_map_error(const std::string& filename) {
{ return std::runtime_error(fmt::format("YAML: {}: trying to dereference non-map node.", filename));
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
if(it!=vars.end())
{
readVals.push_back(name);
return (*it).second;
}
else {
std::cerr << "#PARSER: NO VARIABLE WITH NAME " << name << " FOUND!" << std::endl;
return "";
}
} }
std::string parser::value_or_default(std::string name, std::string defa) static std::runtime_error key_error(const std::string& filename, const std::string& name) {
{ return std::runtime_error(fmt::format("YAML: {}: could not find required key '{}'", filename, name));
std::map<std::string,std::string>::iterator it;
it=vars.find(name);
if(it!=vars.end()) {
readVals.push_back(name);
return (*it).second;
}
else return defa;
} }
bool parser::defined(std::string name)
{ parser::iterator parser::iterator::operator++() {
std::map<std::string,std::string>::iterator it; return iterator{filename_, it_++};
it=vars.find(name);
std::map<std::string,std::vector<std::string> >::iterator arrit = arrs.find(name);
return it != vars.end() || arrit != arrs.end();
// if(it==vars.end()) return false;
// else return true;
}
void parser :: add_comment(std::string new_comm)
{
comment += new_comm;
} }
void parser :: add_delim(std::string new_del) bool parser::iterator::operator!=(const iterator& b) {
{ return it_ != b.it_;
notfine += new_del;
} }
void parser :: get_all(std::ostream& os) parser::parser(const YAML::Node& node, const std::string& filename)
{ : content_{node}, filename_{filename} {
std::map<std::string,std::string>::iterator i; if(!content_.IsMap()) {
std::map<std::string,std::vector<std::string> >::iterator i2; throw non_map_error(filename);
for(i=vars.begin(); i!=vars.end(); i++)
os<<(*i).first<<" = "<<(*i).second<< std::endl;
for(i2=arrs.begin(); i2!=arrs.end();i2++) {
os<<(*i2).first<<"\n";
int size=((*i2).second).size();
for(int j=0;j<size;j++)
os<< ((*i2).second)[j] <<"\n";
os<<(*i2).first<< std::endl;
}
}
void parser :: get_all_with_one_from_specified_array(std::string sfai, int fai, std::ostream& os)
{
std::map<std::string,std::string>::iterator i;
std::map<std::string,std::vector<std::string> >::iterator i2;
for(i=vars.begin(); i!=vars.end(); i++)
os<<(*i).first<<" = "<<(*i).second<<"\n";
for(i2=arrs.begin(); i2!=arrs.end();i2++) {
if((*i2).first==sfai)
os << (*i2).first.substr(1) << " = " << ((*i2).second)[fai] << "\n";
else {
os<<(*i2).first<<"\n";
int size=((*i2).second).size();
for(int j=0;j<size;j++)
os<< ((*i2).second)[j] <<"\n";
os<<(*i2).first<<"\n";
}
} }
} }
std::vector<std::string> parser::unused_parameters() { parser::parser(const std::string& filename)
std::vector<std::string> unused; : parser{YAML::LoadFile(filename), filename} {
std::vector<std::string>::iterator re; }
bool in;
for (std::map<std::string, std::string>::iterator va = vars.begin(); va != vars.end(); ++va) { parser::iterator parser::begin() {
in = false; if(!content_.IsMap()) {
for (re = readVals.begin(); re != readVals.end(); ++re) { throw non_map_error(filename_);
if (va->first == (*re)) {
in = true;
break;
}
}
if (!in)
unused.push_back(va->first);
} }
for (std::map<std::string, std::vector<std::string> >::iterator va = arrs.begin(); va != arrs.end(); ++va) {
in = false; return iterator{filename_, content_.begin()};
for (re = readVals.begin(); re != readVals.end(); ++re) { }
if (va->first == (*re)) {
in = true; parser::iterator parser::end() {
break;
} return iterator{filename_, content_.begin()};
} }
if (!in)
unused.push_back(va->first); bool parser::defined(const std::string& name) {
if(!content_.IsMap()) {
return false;
} }
return unused; return content_[name].IsDefined();
} }
parser parser::operator[](const std::string& name) {
if(!content_.IsMap()) {
throw non_map_error(filename_);
}
auto node = content_[name];
if(!node.IsDefined())
throw key_error(filename_, name);
if(!node.IsMap())
throw std::runtime_error(fmt::format("YAML: {}: Found key '{}', but it has a scalar value. Was expecting it to be a map", filename_, name));
try {
return parser{node, filename_};
} catch(YAML::Exception) {
throw key_error(filename_, name);
}
}
#ifndef MY_PARSER_H #pragma once
#define MY_PARSER_H
#include <yaml-cpp/yaml.h>
#include <iostream> #include <fmt/format.h>
#include <vector>
#include <string> // This is mostly a wrapper around yaml-cpp with more helpful error handling.
#include <sstream> // For simplicity it does not support advanced yaml features such as complex-typed
#include <map> // keys in maps.
class parser class parser {
{ private:
YAML::Node content_;
const std::string filename_;
// fake parser based on a subnode
parser(const YAML::Node& node, const std::string& filename);
public:
class iterator {
private:
std::string filename_;
YAML::Node::iterator it_;
public: public:
parser(); iterator(std::string filename, YAML::Node::iterator it);
parser(std::string); std::pair<std::string, parser> operator*();
iterator operator++();
// one can add other symbols for comments, etc. bool operator!=(const iterator& b);
void add_comment(std::string); };
void add_delim(std::string);
parser(const std::string& filename);
// read a file (if it has not happend yet)
bool read_file(std::string); template<typename T>
T get(const std::string& name) {
// writes out all variables to the specified output if(!content_[name]) {
void get_all(std::ostream&);