Aufgrund eines Security Updates wird GitLab heute zwischen 14:30 und 15:00 Uhr kurzzeitig nicht zur Verfügung stehen. / Due to a security update GitLab will be temporarily unavailable between 2:30 and 3 today.

Commit 916000d0 authored by Lukas Weber's avatar Lukas Weber

switch from yaml to json for speed reasons

parent fcf08d9b
...@@ -9,7 +9,7 @@ project('load-leveller', 'c', 'cpp', ...@@ -9,7 +9,7 @@ project('load-leveller', 'c', 'cpp',
) )
fmt_dep = dependency('fmt', fallback : ['fmt', 'fmt_dep']) fmt_dep = dependency('fmt', fallback : ['fmt', 'fmt_dep'])
yamlcpp_dep = dependency('yaml-cpp', version : '>=0.6.0', fallback : ['yaml-cpp', 'yamlcpp_dep']) json_dep = dependency('nlohmann_json', fallback : ['nlohmann_json', 'nlohmann_json_dep'])
mpi_dep = dependency('mpi', language : 'cpp') mpi_dep = dependency('mpi', language : 'cpp')
# HDF5 is a pain # HDF5 is a pain
...@@ -24,7 +24,7 @@ endif ...@@ -24,7 +24,7 @@ endif
should_install = not meson.is_subproject() should_install = not meson.is_subproject()
loadleveller_deps = [ fmt_dep, yamlcpp_dep, mpi_dep, hdf5_dep ] loadleveller_deps = [ fmt_dep, json_dep, mpi_dep, hdf5_dep ]
subdir('src') subdir('src')
......
...@@ -8,7 +8,7 @@ import os ...@@ -8,7 +8,7 @@ import os
parser = argparse.ArgumentParser(description='Helper script for running and managing loadleveller Monte Carlo jobs.', usage='''loadl <command> <jobscript> [<args>] parser = argparse.ArgumentParser(description='Helper script for running and managing loadleveller Monte Carlo jobs.', usage='''loadl <command> <jobscript> [<args>]
<jobscript> is an executable that prints the job parameter YAML-file to stdout. It is convenient to use the taskmaker python module for this purpose. <jobscript> is an executable that prints the job parameter JSON-file to stdout. It is convenient to use the taskmaker python module for this purpose.
Possible commands and their shorthands are Possible commands and their shorthands are
delete, d delete all data related to a job delete, d delete all data related to a job
...@@ -73,7 +73,7 @@ def run(): ...@@ -73,7 +73,7 @@ def run():
def delete(): def delete():
import shutil import shutil
datadir = '{}.data'.format(job.jobname) datadir = '{}.data'.format(job.jobname)
results_file = '{}.results.yml'.format(job.jobname) results_file = '{}.results.json'.format(job.jobname)
if os.path.exists(datadir): if os.path.exists(datadir):
print('$ rm -r {}'.format(datadir)) print('$ rm -r {}'.format(datadir))
......
import yaml import json
import os import os
import subprocess import subprocess
import errno import errno
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
'''Helpers for handling loadleveller jobfiles/scripts. For lack of a better idea, the job description files of loadleveller are actually executables that output a more verbose yaml parameter file to stdout. Use the taskmaker module to write the input scripts.''' '''Helpers for handling loadleveller jobfiles/scripts. For lack of a better idea, the job description files of loadleveller are actually executables that output a more verbose json parameter file to stdout. Use the taskmaker module to write the input scripts.'''
class JobFileGenError(Exception): class JobFileGenError(Exception):
pass pass
...@@ -24,7 +20,7 @@ class JobFile: ...@@ -24,7 +20,7 @@ class JobFile:
raise JobFileGenError('Generation script "{}" had a non-zero return code. Treating as error.'.format(filename)) raise JobFileGenError('Generation script "{}" had a non-zero return code. Treating as error.'.format(filename))
try: try:
parsed_job = yaml.load(self.raw_jobfile, Loader=SafeLoader) parsed_job = json.loads(self.raw_jobfile)
self.__dict__.update(parsed_job) self.__dict__.update(parsed_job)
except Exception as e: except Exception as e:
raise JobFileGenError('Could not parse job generation script output: {}'.format(e)) raise JobFileGenError('Could not parse job generation script output: {}'.format(e))
...@@ -37,10 +33,10 @@ class JobFile: ...@@ -37,10 +33,10 @@ class JobFile:
except OSError as e: except OSError as e:
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
job_input_filename = os.path.join(datadir, 'parameters.yml') job_input_filename = os.path.join(datadir, 'parameters.json')
with open(job_input_filename, 'w') as f: with open(job_input_filename, 'w') as f:
f.write(self.raw_jobfile) f.write(self.raw_jobfile)
except Exception as e: except Exception as e:
raise JobFileGenError('Could not write parameters.yml: {}'.format(e)) raise JobFileGenError('Could not write parameters.json: {}'.format(e))
return job_input_filename return job_input_filename
...@@ -52,7 +52,7 @@ def job_need_merge(jobfile): ...@@ -52,7 +52,7 @@ def job_need_merge(jobfile):
result_mtime = 0 result_mtime = 0
try: try:
result_mtime = os.path.getmtime(jobfile.jobname+'.results.yml') result_mtime = os.path.getmtime(jobfile.jobname+'.results.json')
except FileNotFoundError: except FileNotFoundError:
return True return True
......
import yaml import json
import numpy as np import numpy as np
import itertools import itertools
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
'''This module can be used to easily extract Monte Carlo results from the *.results.yml file produced by the loadleveller library.''' '''This module can be used to easily extract Monte Carlo results from the *.results.json file produced by the loadleveller library.'''
class Observable: class Observable:
def __init__(self, num_tasks): def __init__(self, num_tasks):
...@@ -21,7 +17,7 @@ class Observable: ...@@ -21,7 +17,7 @@ class Observable:
class MCArchive: class MCArchive:
def __init__(self, filename): def __init__(self, filename):
with open(filename, 'r') as f: with open(filename, 'r') as f:
doc = yaml.load(f, Loader=SafeLoader) doc = json.load(f)
param_names = set(sum([list(task['parameters'].keys()) for task in doc], [])) param_names = set(sum([list(task['parameters'].keys()) for task in doc], []))
observable_names = set(sum([list(task['results'].keys()) for task in doc], [])) observable_names = set(sum([list(task['results'].keys()) for task in doc], []))
...@@ -36,9 +32,9 @@ class MCArchive: ...@@ -36,9 +32,9 @@ class MCArchive:
for obs, value in task['results'].items(): for obs, value in task['results'].items():
o = self.observables[obs] o = self.observables[obs]
o.rebinning_bin_length[i] = int(value.get('rebinning_bin_length',0)) o.rebinning_bin_length[i] = int(value.get('rebin_len',0))
o.rebinning_bin_count[i] = int(value.get('rebinning_bin_count',0)) o.rebinning_bin_count[i] = int(value.get('rebin_count',0))
o.autocorrelation_time[i] = float(value.get('autocorrelation_time',0)) o.autocorrelation_time[i] = float(value.get('autocorr_time',0))
o.mean[i] = np.array(value['mean'], dtype=float) o.mean[i] = np.array(value['mean'], dtype=float)
o.error[i] = np.array(value['error'], dtype=float) o.error[i] = np.array(value['error'], dtype=float)
......
import sys import sys
import os import os
import yaml import yaml
import json
import numpy import numpy
try: try:
...@@ -46,4 +47,4 @@ class TaskMaker: ...@@ -46,4 +47,4 @@ class TaskMaker:
task_dict[k] = v task_dict[k] = v
jobfile_dict['tasks'][task_name] = task_dict jobfile_dict['tasks'][task_name] = task_dict
print(yaml.dump(jobfile_dict, Dumper=SafeDumper)) json.dump(jobfile_dict, sys.stdout, indent=1)
...@@ -129,26 +129,32 @@ int jobinfo::read_dump_progress(int task_id) const { ...@@ -129,26 +129,32 @@ int jobinfo::read_dump_progress(int task_id) const {
} }
void jobinfo::concatenate_results() { void jobinfo::concatenate_results() {
std::ofstream cat_results{fmt::format("{}.results.yml", jobname)}; std::ofstream cat_results{fmt::format("{}.results.json", jobname)};
cat_results << "[";
for(size_t i = 0; i < task_names.size(); i++) { for(size_t i = 0; i < task_names.size(); i++) {
std::ifstream res_file{taskdir(i) + "/results.yml"}; std::ifstream res_file{taskdir(i) + "/results.json"};
res_file.seekg(0, res_file.end); res_file.seekg(0, res_file.end);
size_t size = res_file.tellg(); size_t size = res_file.tellg();
res_file.seekg(0, res_file.beg); res_file.seekg(0, res_file.beg);
std::vector<char> buf(size + 1, 0); std::vector<char> buf(size + 1, 0);
res_file.read(buf.data(), size); res_file.read(buf.data(), size);
cat_results << buf.data() << "\n"; cat_results << buf.data();
if(i < task_names.size()-1) {
cat_results << ",";
}
cat_results << "\n";
} }
cat_results << "]\n";
} }
void jobinfo::merge_task(int task_id, const std::vector<evalable> &evalables) { void jobinfo::merge_task(int task_id, const std::vector<evalable> &evalables) {
std::vector<std::string> meas_files = list_run_files(taskdir(task_id), "meas\\.h5"); std::vector<std::string> meas_files = list_run_files(taskdir(task_id), "meas\\.h5");
results results = merge(meas_files, evalables); results results = merge(meas_files, evalables);
std::string result_filename = fmt::format("{}/results.yml", taskdir(task_id)); std::string result_filename = fmt::format("{}/results.json", taskdir(task_id));
const std::string &task_name = task_names.at(task_id); const std::string &task_name = task_names.at(task_id);
results.write_yaml(result_filename, taskdir(task_id), jobfile["tasks"][task_name].get_yaml()); results.write_yaml(result_filename, taskdir(task_id), jobfile["tasks"][task_name].get_json());
} }
void jobinfo::log(const std::string &message) { void jobinfo::log(const std::string &message) {
......
#include "parser.h" #include "parser.h"
#include <fstream>
namespace loadl { namespace loadl {
parser::iterator::iterator(std::string filename, YAML::Node::iterator it) parser::iterator::iterator(std::string filename, json::iterator it)
: filename_{std::move(filename)}, it_{std::move(it)} {} : filename_{std::move(filename)}, it_{std::move(it)} {}
std::pair<std::string, parser> parser::iterator::operator*() { std::pair<std::string, parser> parser::iterator::operator*() {
try { return std::make_pair(it_.key(), parser{it_.value(), filename_});
return std::make_pair(it_->first.as<std::string>(), parser{it_->second, filename_});
} catch(YAML::Exception &e) {
throw std::runtime_error(
fmt::format("YAML: {}: dereferencing map key failed: {}. Maybe it was not a string?",
filename_, e.what()));
}
} }
static std::runtime_error non_map_error(const std::string &filename) { static std::runtime_error non_map_error(const std::string &filename) {
return std::runtime_error( return std::runtime_error(
fmt::format("YAML: {}: trying to dereference non-map node.", filename)); fmt::format("json: {}: trying to dereference non-map node.", filename));
} }
static std::runtime_error key_error(const std::string &filename, const std::string &key) { static std::runtime_error key_error(const std::string &filename, const std::string &key) {
return std::runtime_error( return std::runtime_error(
fmt::format("YAML: {}: could not find required key '{}'", filename, key)); fmt::format("json: {}: could not find required key '{}'", filename, key));
} }
parser::iterator parser::iterator::operator++() { parser::iterator parser::iterator::operator++() {
...@@ -33,17 +28,23 @@ bool parser::iterator::operator!=(const iterator &b) { ...@@ -33,17 +28,23 @@ bool parser::iterator::operator!=(const iterator &b) {
return it_ != b.it_; return it_ != b.it_;
} }
parser::parser(const YAML::Node &node, const std::string &filename) parser::parser(const json &node, const std::string &filename)
: content_{node}, filename_{filename} { : content_(node), filename_{filename} {
if(!content_.IsMap()) { if(!content_.is_object()) {
throw non_map_error(filename); throw non_map_error(filename);
} }
} }
parser::parser(const std::string &filename) : parser{YAML::LoadFile(filename), filename} {} parser::parser(const std::string &filename) : filename_{filename} {
std::ifstream f(filename);
f >> content_;
if(!content_.is_object()) {
throw non_map_error(filename);
}
}
parser::iterator parser::begin() { parser::iterator parser::begin() {
if(!content_.IsMap()) { if(!content_.is_object()) {
throw non_map_error(filename_); throw non_map_error(filename_);
} }
...@@ -55,35 +56,31 @@ parser::iterator parser::end() { ...@@ -55,35 +56,31 @@ parser::iterator parser::end() {
} }
bool parser::defined(const std::string &key) const { bool parser::defined(const std::string &key) const {
if(!content_.IsMap()) { if(!content_.is_object()) {
return false; return false;
} }
return content_[key].IsDefined(); return content_.find(key) != content_.end();
} }
parser parser::operator[](const std::string &key) { parser parser::operator[](const std::string &key) {
if(!content_.IsMap()) { if(!content_.is_object()) {
throw non_map_error(filename_); throw non_map_error(filename_);
} }
auto node = content_[key]; auto node = content_[key];
if(!node.IsDefined()) { if(node.is_null()) {
throw key_error(filename_, key); throw key_error(filename_, key);
} }
if(!node.IsMap()) { if(!node.is_object()) {
throw std::runtime_error(fmt::format( throw std::runtime_error(fmt::format(
"YAML: {}: Found key '{}', but it has a scalar value. Was expecting it to be a map", "json: {}: Found key '{}', but it has a scalar value. Was expecting it to be a map",
filename_, key)); filename_, key));
} }
try { return parser{node, filename_};
return parser{node, filename_};
} catch(YAML::Exception &) {
throw key_error(filename_, key);
}
} }
const YAML::Node &parser::get_yaml() { const json &parser::get_json() const {
return content_; return content_;
} }
} }
#pragma once #pragma once
#include <fmt/format.h> #include <fmt/format.h>
#include <yaml-cpp/yaml.h> #include <nlohmann/json.hpp>
namespace loadl { namespace loadl {
...@@ -9,22 +9,24 @@ namespace loadl { ...@@ -9,22 +9,24 @@ namespace loadl {
// For simplicity it does not support advanced yaml features such as complex-typed // For simplicity it does not support advanced yaml features such as complex-typed
// keys in maps. // keys in maps.
using json = nlohmann::json;
class parser { class parser {
private: private:
YAML::Node content_; json content_;
const std::string filename_; const std::string filename_;
// fake parser based on a subnode // fake parser based on a subnode
parser(const YAML::Node &node, const std::string &filename); parser(const json &node, const std::string &filename);
public: public:
class iterator { class iterator {
private: private:
std::string filename_; std::string filename_;
YAML::Node::iterator it_; json::iterator it_;
public: public:
iterator(std::string filename, YAML::Node::iterator it); iterator(std::string filename, json::iterator it);
std::pair<std::string, parser> operator*(); std::pair<std::string, parser> operator*();
iterator operator++(); iterator operator++();
bool operator!=(const iterator &b); bool operator!=(const iterator &b);
...@@ -34,16 +36,17 @@ public: ...@@ -34,16 +36,17 @@ public:
template<typename T> template<typename T>
T get(const std::string &key) const { T get(const std::string &key) const {
if(!content_[key]) { auto v = content_.find(key);
if(v == content_.end()) {
throw std::runtime_error( throw std::runtime_error(
fmt::format("YAML: {}: required key '{}' not found.", filename_, key)); fmt::format("json: {}: required key '{}' not found.", filename_, key));
} }
return content_[key].as<T>(); return *v;
} }
template<typename T> template<typename T>
auto get(const std::string &key, T default_val) const { auto get(const std::string &key, T default_val) const {
return content_[key].as<T>(default_val); return content_.value<T>(key, default_val);
} }
// is key defined? // is key defined?
...@@ -55,6 +58,6 @@ public: ...@@ -55,6 +58,6 @@ public:
// This gives access to the underlying yaml-cpp api. Only use it if you absolutely need to. // This gives access to the underlying yaml-cpp api. Only use it if you absolutely need to.
// This function is needed to dump the task settings into the result file for example. // This function is needed to dump the task settings into the result file for example.
const YAML::Node &get_yaml(); const json &get_json() const;
}; };
} }
#include "results.h" #include "results.h"
#include <algorithm> #include <algorithm>
#include <fstream> #include <fstream>
#include <yaml-cpp/yaml.h> #include <nlohmann/json.hpp>
namespace loadl { namespace loadl {
void results::write_yaml(const std::string &filename, const std::string &taskdir, void results::write_json(const std::string &filename, const std::string &taskdir,
const YAML::Node &params) { const nlohmann::json &params) {
YAML::Emitter out; using json = nlohmann::json;
out << YAML::BeginSeq;
out << YAML::BeginMap; json obs_list;
out << YAML::Key << "task" << YAML::Value << taskdir;
out << YAML::Key << "parameters" << YAML::Value << params;
out << YAML::Key << "results" << YAML::Value << YAML::BeginMap;
for(auto &[obs_name, obs] : observables) { for(auto &[obs_name, obs] : observables) {
out << YAML::Key << obs_name;
if(obs.internal_bin_length == 0) {
out << YAML::Comment("evalable");
}
out << YAML::Value << YAML::BeginMap;
out << YAML::Key << "rebinning_bin_length" << YAML::Value << obs.rebinning_bin_length;
out << YAML::Key << "rebinning_bin_count" << YAML::Value << obs.rebinning_bin_count;
out << YAML::Key << "internal_bin_length" << YAML::Value << obs.internal_bin_length;
double max_auto_time = 0; double max_auto_time = 0;
if(obs.autocorrelation_time.size() > 0) { if(obs.autocorrelation_time.size() > 0) {
max_auto_time = max_auto_time =
*std::max_element(obs.autocorrelation_time.begin(), obs.autocorrelation_time.end()); *std::max_element(obs.autocorrelation_time.begin(), obs.autocorrelation_time.end());
} }
out << YAML::Key << "autocorrelation_time" << YAML::Value << max_auto_time;
out << YAML::Key << "mean" << YAML::Value << obs.mean; obs_list[obs_name] = {
out << YAML::Key << "error" << YAML::Value << obs.error; {"rebin_len", obs.rebinning_bin_length},
out << YAML::EndMap; {"rebin_count", obs.rebinning_bin_count},
{"internal_bin_len", obs.internal_bin_length},
{"autocorr_time", max_auto_time},
{"mean", obs.mean},
{"error", obs.error},
};
} }
out << YAML::EndMap;
out << YAML::EndMap; nlohmann::json out = {
out << YAML::EndSeq; {"task", taskdir},
{"parameters", params},
{"results", obs_list}
};
std::ofstream file(filename); std::ofstream file(filename);
file << out.c_str(); file << out.dump(1);
} }
} }
#pragma once #pragma once
#include <map> #include <map>
#include <vector> #include <vector>
#include <nlohmann/json.hpp>
namespace YAML {
class Node;
}
namespace loadl { namespace loadl {
...@@ -35,8 +32,8 @@ struct observable_result { ...@@ -35,8 +32,8 @@ struct observable_result {
struct results { struct results {
std::map<std::string, observable_result> observables; std::map<std::string, observable_result> observables;
// writes out the results in a yaml file. // writes out the results in a json file.
void write_yaml(const std::string &filename, const std::string &taskdir, void write_json(const std::string &filename, const std::string &taskdir,
const YAML::Node &params); const nlohmann::json &params);
}; };
} }
...@@ -264,18 +264,14 @@ void runner_pt_master::checkpoint_read() { ...@@ -264,18 +264,14 @@ void runner_pt_master::checkpoint_read() {
} }
} }
void runner_pt_master::write_params_yaml() { void runner_pt_master::write_params_json() {
using namespace YAML; nlohmann::json params;
Emitter params;
params << BeginMap;
for(auto c : pt_chains_) { for(auto c : pt_chains_) {
params << Key << fmt::format("chain{:04d}", c.id); params[fmt::format("chain{:04d}", c.id)] = c.params;
params << Value << Flow << c.params;
} }
params << EndMap;
std::ofstream file{job_.jobdir() + "/pt_optimized_params.yml"}; std::ofstream file{job_.jobdir() + "/pt_optimized_params.json"};
file << params.c_str() << "\n"; file << params.dump(1) << "\n";
} }
void runner_pt_master::write_param_optimization_stats() { void runner_pt_master::write_param_optimization_stats() {
...@@ -321,7 +317,7 @@ void runner_pt_master::checkpoint_write() { ...@@ -321,7 +317,7 @@ void runner_pt_master::checkpoint_write() {
} }
if(po_config_.enabled) { if(po_config_.enabled) {
write_params_yaml(); write_params_json();
} }
} }
......
...@@ -76,7 +76,7 @@ private: ...@@ -76,7 +76,7 @@ private:
void construct_pt_chains(); void construct_pt_chains();
void checkpoint_write(); void checkpoint_write();
void checkpoint_read(); void checkpoint_read();
void write_params_yaml(); void write_params_json();
void write_param_optimization_stats(); void write_param_optimization_stats();
int schedule_chain_run(); int schedule_chain_run();
......
[wrap-file]
directory = nlohmann_json-3.4.0
lead_directory_missing = true
source_url = https://github.com/nlohmann/json/releases/download/v3.4.0/include.zip
source_filename = nlohmann_json-3.4.0.zip
source_hash = bfec46fc0cee01c509cf064d2254517e7fa80d1e7647fea37cf81d97c5682bdc
patch_url = https://wrapdb.mesonbuild.com/v1/projects/nlohmann_json/3.4.0/2/get_zip
patch_filename = nlohmann_json-3.4.0-2-wrap.zip
patch_hash = ce65c4827a8c19e36539ca48d713c5c327f9788ef99d578a996bc136a8619d75
[wrap-git]
directory = yaml-cpp
url = https://github.com/lukas-weber/yaml-cpp.git
revision = master