jobinfo.cpp 4.7 KB
Newer Older
1
#include "jobinfo.h"
2
#include "mc.h"
Lukas Weber's avatar
Lukas Weber committed
3
#include "merger.h"
4
#include <ctime>
Lukas Weber's avatar
Lukas Weber committed
5
#include <filesystem>
Lukas Weber's avatar
Lukas Weber committed
6
#include <fstream>
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#include <iomanip>
#include <iostream>
#include <regex>

namespace loadl {

// parses the duration '[[hours:]minutes:]seconds' into seconds
// replace as soon as there is an alternative
static int parse_duration(const std::string &str) {
	size_t idx;

	try {
		int i1 = std::stoi(str, &idx, 10);
		if(idx == str.size()) {
			return i1;
		} else if(str[idx] == ':') {
			std::string str1 = str.substr(idx + 1);
			int i2 = std::stoi(str1, &idx, 10);

			if(idx == str1.size()) {
				return 60 * i1 + i2;
			} else if(str[idx] == ':') {
				std::string str2 = str1.substr(idx + 1);
				int i3 = std::stoi(str2, &idx, 10);
				if(idx != str2.size()) {
					throw std::runtime_error{"minutes"};
				}

				return 60 * 60 * i1 + 60 * i2 + i3;
			} else {
				throw std::runtime_error{"hours"};
			}
		} else {
			throw std::runtime_error{"seconds"};
		}
	} catch(std::exception &e) {
		throw std::runtime_error{fmt::format(
		    "'{}' does not fit time format [[hours:]minutes:]seconds: {}", str, e.what())};
	}
}

48
49
std::filesystem::path jobinfo::taskdir(int task_id) const {
	return jobdir / task_names.at(task_id);
50
51
}

52
std::filesystem::path jobinfo::rundir(int task_id, int run_id) const {
Lukas Weber's avatar
Lukas Weber committed
53
	return taskdir(task_id) / fmt::format("run{:04d}", run_id);
54
55
}

56
jobinfo::jobinfo(const std::filesystem::path &jobfile_name, register_evalables_func evalable_func)
Lukas Weber's avatar
Lukas Weber committed
57
    : evalable_func_{evalable_func}, jobfile{jobfile_name}, jobdir{jobfile_name.parent_path()} {
58
59
60
61
62
63
64
65
	for(auto node : jobfile["tasks"]) {
		std::string task_name = node.first;
		task_names.push_back(task_name);
	}
	std::sort(task_names.begin(), task_names.end());

	jobname = jobfile.get<std::string>("jobname");

66
	std::error_code ec;
67
	std::filesystem::create_directories(jobdir, ec);
68
69
70
71

	// perhaps a bit controversally, jobinfo tries to create the task directories. TODO: single file
	// output.
	for(size_t i = 0; i < task_names.size(); i++) {
72
		std::filesystem::create_directories(taskdir(i));
73
74
75
76
	}

	parser jobconfig{jobfile["jobconfig"]};

Lukas Weber's avatar
Lukas Weber committed
77
	runtime = parse_duration(jobconfig.get<std::string>("mc_runtime"));
78
79
80
81
82
83
	checkpoint_time = parse_duration(jobconfig.get<std::string>("mc_checkpoint_time"));
}

// This function lists files that could be run files being in the taskdir
// and having the right file_ending.
// The regex has to be matched with the output of the rundir function.
84
std::vector<std::filesystem::path> jobinfo::list_run_files(const std::string &taskdir,
Lukas Weber's avatar
Lukas Weber committed
85
                                                           const std::string &file_ending) {
Lukas Weber's avatar
Lukas Weber committed
86
	std::regex run_filename{"^run\\d{4,}\\." + file_ending + "$"};
87
	std::vector<std::filesystem::path> results;
88
89
90
91

	for(const auto &p : std::filesystem::directory_iterator(taskdir)) {
		if(std::regex_search(p.path().filename().string(), run_filename)) {
			results.emplace_back(p.path());
92
93
94
95
96
97
98
		}
	}

	return results;
}

int jobinfo::read_dump_progress(int task_id) const {
Lukas Weber's avatar
Lukas Weber committed
99
	size_t sweeps = 0;
100
101
	try {
		for(auto &dump_name : list_run_files(taskdir(task_id), "dump\\.h5")) {
Lukas Weber's avatar
Lukas Weber committed
102
			size_t dump_sweeps = 0;
103
104
105
106
107
108
109
110
111
112
113
114
			iodump d = iodump::open_readonly(dump_name);
			d.get_root().read("sweeps", dump_sweeps);
			sweeps += dump_sweeps;
		}
	} catch(std::ios_base::failure &e) {
		// might happen if the taskdir does not exist
	}

	return sweeps;
}

void jobinfo::concatenate_results() {
Lukas Weber's avatar
Lukas Weber committed
115
	std::ofstream cat_results{jobdir.parent_path() / fmt::format("{}.results.json", jobname)};
116
	cat_results << "[";
117
	for(size_t i = 0; i < task_names.size(); i++) {
118
		std::ifstream res_file{taskdir(i) / "results.json"};
119
120
121
122
123
124
		res_file.seekg(0, res_file.end);
		size_t size = res_file.tellg();
		res_file.seekg(0, res_file.beg);

		std::vector<char> buf(size + 1, 0);
		res_file.read(buf.data(), size);
125
		cat_results << buf.data();
126
		if(i < task_names.size() - 1) {
127
128
129
			cat_results << ",";
		}
		cat_results << "\n";
130
	}
131
	cat_results << "]\n";
132
133
}

134
void jobinfo::merge_task(int task_id) {
135
	std::vector<std::filesystem::path> meas_files = list_run_files(taskdir(task_id), "meas\\.h5");
136
137
	size_t rebinning_bin_length = jobfile["jobconfig"].get<size_t>("merge_rebin_length", 0);
	size_t sample_skip = jobfile["jobconfig"].get<size_t>("merge_sample_skip", 0);
138
139
140
	results results = merge(meas_files, rebinning_bin_length, sample_skip);

	evaluator eval{results};
141
	evalable_func_(eval, jobfile["tasks"][task_names[task_id]]);
142
	eval.append_results();
143

144
	std::filesystem::path result_filename = taskdir(task_id) / "results.json";
145
	const std::string &task_name = task_names.at(task_id);
Lukas Weber's avatar
...    
Lukas Weber committed
146
	results.write_json(result_filename, taskdir(task_id), jobfile["tasks"][task_name].get_json());
147
148
149
150
151
152
153
154
}

void jobinfo::log(const std::string &message) {
	std::time_t t = std::time(nullptr);
	std::cout << std::put_time(std::localtime(&t), "%F %T: ") << message << "\n";
}

}