Commit 4828f95b authored by Lukas Weber's avatar Lukas Weber
Browse files

testing and implementing missing features/tools

parent 1cdff5f2
...@@ -21,6 +21,7 @@ set(SRCs ...@@ -21,6 +21,7 @@ set(SRCs
mc.cpp mc.cpp
runner_single.cpp runner_single.cpp
runner_task.cpp runner_task.cpp
results.cpp
) )
add_library(load_leveller ${SRCs}) add_library(load_leveller ${SRCs})
......
...@@ -3,20 +3,17 @@ ...@@ -3,20 +3,17 @@
#include <unistd.h> #include <unistd.h>
#include <sstream> #include <sstream>
#include <iostream> #include <iostream>
#include <sys/file.h>
static bool filter_available(H5Z_filter_t filter) { static bool filter_available(H5Z_filter_t filter) {
htri_t avail = H5Zfilter_avail(filter); htri_t avail = H5Zfilter_avail(filter);
if(not avail) { if(avail == 0) {
return false; return false;
} }
unsigned int filter_info; unsigned int filter_info;
herr_t status = H5Zget_filter_info(filter, &filter_info); herr_t status = H5Zget_filter_info(filter, &filter_info);
if(status < 0 or not (filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) or not (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)) { return status >= 0 && (filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) != 0 && (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED) != 0;
return false;
}
return true;
} }
static herr_t H5Ewalk_cb(unsigned int n, const H5E_error2_t *err_desc, void *client_data) { static herr_t H5Ewalk_cb(unsigned int n, const H5E_error2_t *err_desc, void *client_data) {
...@@ -25,8 +22,8 @@ static herr_t H5Ewalk_cb(unsigned int n, const H5E_error2_t *err_desc, void *cli ...@@ -25,8 +22,8 @@ static herr_t H5Ewalk_cb(unsigned int n, const H5E_error2_t *err_desc, void *cli
char *min_str = H5Eget_minor(err_desc->min_num); char *min_str = H5Eget_minor(err_desc->min_num);
char *maj_str = H5Eget_major(err_desc->maj_num); char *maj_str = H5Eget_major(err_desc->maj_num);
s << fmt::format("#{}: {}:{} in {}(): {}\n", n, err_desc->file_name, err_desc->line, err_desc->func_name, err_desc->desc); s << fmt::format("#{}: {}:{} in {}(): {}\n", n, err_desc->file_name, err_desc->line, err_desc->func_name, err_desc->desc);
s << fmt::format(" {}: {}\n", err_desc->maj_num, maj_str); s << fmt::format(" {}: {}\n", static_cast<int>(err_desc->maj_num), maj_str);
s << fmt::format(" {}: {}\n\n", err_desc->min_num, min_str); s << fmt::format(" {}: {}\n\n", static_cast<int>(err_desc->min_num), min_str);
free(min_str); free(min_str);
free(maj_str); free(maj_str);
...@@ -57,8 +54,9 @@ iodump::group::group(hid_t parent, const std::string& path) { ...@@ -57,8 +54,9 @@ iodump::group::group(hid_t parent, const std::string& path) {
} }
iodump::group::~group() { iodump::group::~group() {
if(group_ < 0) if(group_ < 0) {
return; return;
}
herr_t status = H5Gclose(group_); herr_t status = H5Gclose(group_);
if(status < 0) { if(status < 0) {
std::cerr << iodump_exception{"H5Gclose"}.what(); std::cerr << iodump_exception{"H5Gclose"}.what();
...@@ -72,10 +70,11 @@ iodump::group::iterator iodump::group::begin() const { ...@@ -72,10 +70,11 @@ iodump::group::iterator iodump::group::begin() const {
} }
iodump::group::iterator iodump::group::end() const { iodump::group::iterator iodump::group::end() const {
H5G_info_t info; H5G_info_t info{};
herr_t status = H5Gget_info(group_, &info); herr_t status = H5Gget_info(group_, &info);
if(status < 0) if(status < 0) {
throw iodump_exception{"H5Gget_info"}; throw iodump_exception{"H5Gget_info"};
}
return iodump::group::iterator{group_, info.nlinks}; return iodump::group::iterator{group_, info.nlinks};
} }
...@@ -86,13 +85,15 @@ iodump::group::iterator::iterator(hid_t group, uint64_t idx) ...@@ -86,13 +85,15 @@ iodump::group::iterator::iterator(hid_t group, uint64_t idx)
std::string iodump::group::iterator::operator*() { std::string iodump::group::iterator::operator*() {
ssize_t name_size = H5Lget_name_by_idx(group_, "/", H5_INDEX_NAME, H5_ITER_INC, idx_, nullptr, 0, H5P_DEFAULT); ssize_t name_size = H5Lget_name_by_idx(group_, "/", H5_INDEX_NAME, H5_ITER_INC, idx_, nullptr, 0, H5P_DEFAULT);
if(name_size < 0) if(name_size < 0) {
throw iodump_exception{"H5Lget_name_by_idx"}; throw iodump_exception{"H5Lget_name_by_idx"};
}
std::vector<char> buf(name_size+1); std::vector<char> buf(name_size+1);
name_size = H5Lget_name_by_idx(group_, "/", H5_INDEX_NAME, H5_ITER_INC, idx_, buf.data(), buf.size(), H5P_DEFAULT); name_size = H5Lget_name_by_idx(group_, "/", H5_INDEX_NAME, H5_ITER_INC, idx_, buf.data(), buf.size(), H5P_DEFAULT);
if(name_size < 0) if(name_size < 0) {
throw iodump_exception{"H5Lget_name_by_idx"}; throw iodump_exception{"H5Lget_name_by_idx"};
}
return std::string(buf.data()); return std::string(buf.data());
} }
...@@ -107,38 +108,42 @@ bool iodump::group::iterator::operator!=(const iterator& b) { ...@@ -107,38 +108,42 @@ bool iodump::group::iterator::operator!=(const iterator& b) {
} }
iodump iodump::create(const std::string& filename) { iodump iodump::create(const std::string& filename) {
H5Eset_auto(H5E_DEFAULT, NULL, NULL); H5Eset_auto(H5E_DEFAULT, nullptr, nullptr);
hid_t file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); hid_t file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
if(file < 0) if(file < 0) {
throw iodump_exception{"H5Fcreate"}; throw iodump_exception{"H5Fcreate"};
}
return iodump{file}; return iodump{filename, file};
} }
iodump iodump::open_readonly(const std::string& filename) { iodump iodump::open_readonly(const std::string& filename) {
H5Eset_auto(H5E_DEFAULT, NULL, NULL); H5Eset_auto(H5E_DEFAULT, nullptr, nullptr);
hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
if(file < 0) if(file < 0) {
throw iodump_exception{"H5Fopen"}; throw iodump_exception{"H5Fopen"};
return iodump{file}; }
return iodump{filename, file};
} }
iodump iodump::open_readwrite(const std::string& filename) { iodump iodump::open_readwrite(const std::string& filename) {
H5Eset_auto(H5E_DEFAULT, NULL, NULL); H5Eset_auto(H5E_DEFAULT, nullptr, nullptr);
if(access(filename.c_str(), R_OK) != F_OK) { if(access(filename.c_str(), R_OK) != F_OK) {
create(filename); create(filename);
} }
hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
if(file < 0) if(file < 0) {
throw iodump_exception{"H5Fopen"}; throw iodump_exception{"H5Fopen"};
return iodump{file}; }
return iodump{filename, file};
} }
iodump::iodump(hid_t h5_file) : h5_file_{h5_file} { iodump::iodump(std::string filename, hid_t h5_file) : filename_{std::move(filename)}, h5_file_{h5_file} {
if(compression_filter_ != 0 and not filter_available(compression_filter_)) if(compression_filter_ != 0 && !filter_available(compression_filter_)) {
throw iodump_exception{"H5Filter not available."}; throw iodump_exception{"H5Filter not available."};
}
} }
iodump::~iodump() { iodump::~iodump() {
...@@ -149,59 +154,51 @@ iodump::group iodump::get_root() { ...@@ -149,59 +154,51 @@ iodump::group iodump::get_root() {
return group{h5_file_, "/"}; return group{h5_file_, "/"};
} }
hid_t iodump::group::create_dataset(const std::string& name, hid_t datatype, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited) const { iodump::h5_handle iodump::group::create_dataset(const std::string& name, hid_t datatype, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited) const {
herr_t status; herr_t status;
hid_t dataset;
if(exists(name)) { if(exists(name)) {
dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT); h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
if(dataset < 0) h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset); hsize_t oldsize = H5Sget_simple_extent_npoints(*dataspace);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"}; if(oldsize < 0) {
hsize_t oldsize = H5Sget_simple_extent_npoints(dataspace);
if(oldsize < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"}; throw iodump_exception{"H5Sget_simple_extent_npoints"};
if(oldsize != size) }
if(oldsize != size) {
throw std::runtime_error{"iodump: tried to write into an existing dataset with different dimensions!"}; throw std::runtime_error{"iodump: tried to write into an existing dataset with different dimensions!"};
}
return dataset;
} else { } else {
hid_t dataspace; hid_t dataspace_h;
if(not unlimited) { if(not unlimited) {
dataspace = H5Screate_simple(1, &size, nullptr); dataspace_h = H5Screate_simple(1, &size, nullptr);
} else { } else {
hsize_t maxdim = H5S_UNLIMITED; hsize_t maxdim = H5S_UNLIMITED;
dataspace = H5Screate_simple(1, &size, &maxdim); dataspace_h = H5Screate_simple(1, &size, &maxdim);
} }
h5_handle dataspace{dataspace_h, H5Sclose};
hid_t plist = H5Pcreate(H5P_DATASET_CREATE); h5_handle plist{H5Pcreate(H5P_DATASET_CREATE), H5Pclose};
if(chunk_size > 1) { // do not use compression on small datasets if(chunk_size > 1) { // do not use compression on small datasets
status = H5Pset_chunk(plist, 1, &chunk_size); status = H5Pset_chunk(*plist, 1, &chunk_size);
if(status < 0) if(status < 0) {
throw iodump_exception{"H5Pset_chunk"}; throw iodump_exception{"H5Pset_chunk"};
}
if(compression_filter == H5Z_FILTER_DEFLATE) { if(compression_filter == H5Z_FILTER_DEFLATE) {
status = H5Pset_deflate(plist, 6); status = H5Pset_deflate(*plist, 6);
if(status < 0) if(status < 0) {
throw iodump_exception{"H5Pset_deflate"}; throw iodump_exception{"H5Pset_deflate"};
}
} }
} }
dataset = H5Dcreate2(group_, name.c_str(), datatype, dataspace, H5P_DEFAULT, plist, H5P_DEFAULT); return h5_handle{H5Dcreate2(group_, name.c_str(), datatype, *dataspace, H5P_DEFAULT, *plist, H5P_DEFAULT), H5Dclose};
if(dataset < 0)
throw iodump_exception{"H5Dcreate2"};
status = H5Pclose(plist);
if(status < 0)
throw iodump_exception{"H5Pclose"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
} }
return dataset;
} }
iodump::group iodump::group::open_group(const std::string& path) const { iodump::group iodump::group::open_group(const std::string& path) const {
...@@ -209,17 +206,13 @@ iodump::group iodump::group::open_group(const std::string& path) const { ...@@ -209,17 +206,13 @@ iodump::group iodump::group::open_group(const std::string& path) const {
} }
size_t iodump::group::get_extent(const std::string& name) const { size_t iodump::group::get_extent(const std::string& name) const {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT); h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
if(dataset < 0) h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace); // rank > 1 will get flattened when loaded. int size = H5Sget_simple_extent_npoints(*dataspace); // rank > 1 will get flattened when loaded.
if(size < 0) if(size < 0) {
throw iodump_exception{"H5Sget_simple_extent_npoints"}; throw iodump_exception{"H5Sget_simple_extent_npoints"};
}
return size; return size;
} }
...@@ -228,9 +221,39 @@ bool iodump::group::exists(const std::string &path) const { ...@@ -228,9 +221,39 @@ bool iodump::group::exists(const std::string &path) const {
htri_t exists = H5Lexists(group_, path.c_str(), H5P_DEFAULT); htri_t exists = H5Lexists(group_, path.c_str(), H5P_DEFAULT);
if(exists == 0) { if(exists == 0) {
return false; return false;
} else if(exists < 0) { }
if(exists < 0) {
throw iodump_exception{"H5Lexists"}; throw iodump_exception{"H5Lexists"};
} }
return true; return true;
} }
iodump::h5_handle::h5_handle(hid_t handle, herr_t (*closer)(hid_t))
: closer_{closer}, handle_{handle} {
if(handle < 0) {
throw iodump_exception{"h5_handle"};
}
}
iodump::h5_handle::h5_handle(h5_handle&& h) noexcept
: closer_{h.closer_}, handle_{h.handle_} {
h.handle_ = -1;
}
iodump::h5_handle::~h5_handle() {
if(handle_ < 0) {
return;
}
herr_t status = closer_(handle_);
if(status < 0) {
std::cerr << iodump_exception{"~h5_handle"}.what() << "\n";
std::abort();
}
}
hid_t iodump::h5_handle::operator*() {
return handle_;
}
...@@ -19,7 +19,33 @@ public: ...@@ -19,7 +19,33 @@ public:
// writing depending on how you open it. If you write to a read only file, // writing depending on how you open it. If you write to a read only file,
// there will be an error probably. // there will be an error probably.
class iodump { class iodump {
public: private:
// helper class to make sure those pesky HDF5 hid_t handles are always closed
class h5_handle {
public:
h5_handle(hid_t handle, herr_t (*closer)(hid_t));
~h5_handle();
h5_handle(h5_handle&) = delete;
h5_handle(h5_handle&&) noexcept;
hid_t operator*();
private:
herr_t (*closer_)(hid_t);
hid_t handle_;
};
iodump(std::string filename, hid_t h5_file);
const std::string filename_;
const hid_t h5_file_;
// TODO: make these variable if necessary
static const H5Z_filter_t compression_filter_ = H5Z_FILTER_DEFLATE;
static const size_t chunk_size_ = 1000;
template<typename T>
constexpr static hid_t h5_datatype();
public:
// Wrapper around the concept of a HDF5 group. // Wrapper around the concept of a HDF5 group.
// You can list over the group elements by using the iterators. // You can list over the group elements by using the iterators.
class group { class group {
...@@ -65,7 +91,7 @@ public: ...@@ -65,7 +91,7 @@ public:
// chunk_size == 0 means contiguous storage // chunk_size == 0 means contiguous storage
// if the dataset already exists, we try to overwrite it. However it must have the same extent for that to work. // if the dataset already exists, we try to overwrite it. However it must have the same extent for that to work.
hid_t create_dataset(const std::string& name, hid_t datatype, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited) const; iodump::h5_handle create_dataset(const std::string& name, hid_t datatype, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited) const;
}; };
// delete what was there and create a new file for writing // delete what was there and create a new file for writing
...@@ -78,17 +104,8 @@ public: ...@@ -78,17 +104,8 @@ public:
iodump(iodump&) = delete; iodump(iodump&) = delete;
~iodump(); ~iodump();
private:
iodump(hid_t h5_file);
const hid_t h5_file_;
// TODO: make these variable if necessary friend class group;
static const H5Z_filter_t compression_filter_ = H5Z_FILTER_DEFLATE;
static const size_t chunk_size_ = 1000;
template<typename T>
constexpr static hid_t h5_datatype();
}; };
template<typename T> template<typename T>
...@@ -124,9 +141,6 @@ constexpr hid_t iodump::h5_datatype() { ...@@ -124,9 +141,6 @@ constexpr hid_t iodump::h5_datatype() {
// ... or it is a native datatype I forgot to add. Then add it. // ... or it is a native datatype I forgot to add. Then add it.
} }
template<class T> template<class T>
void iodump::group::write(const std::string& name, const std::vector<T>& data) const { void iodump::group::write(const std::string& name, const std::vector<T>& data) const {
int chunk_size = 0; int chunk_size = 0;
...@@ -138,13 +152,10 @@ void iodump::group::write(const std::string& name, const std::vector<T>& data) c ...@@ -138,13 +152,10 @@ void iodump::group::write(const std::string& name, const std::vector<T>& data) c
compression_filter= iodump::compression_filter_; compression_filter= iodump::compression_filter_;
} }
hid_t dataset = create_dataset(name, h5_datatype<T>(), data.size(), chunk_size, compression_filter, false); h5_handle dataset{create_dataset(name, h5_datatype<T>(), data.size(), chunk_size, compression_filter, false)};
herr_t status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data()); herr_t status = H5Dwrite(*dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data());
if(status < 0) if(status < 0)
throw iodump_exception{"H5Dwrite"}; throw iodump_exception{"H5Dwrite"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
} }
template<class T> template<class T>
...@@ -166,79 +177,58 @@ void iodump::group::insert_back(const std::string& name, const std::vector<T>& d ...@@ -166,79 +177,58 @@ void iodump::group::insert_back(const std::string& name, const std::vector<T>& d
create_dataset(name, h5_datatype<T>(), 0, chunk_size_, compression_filter_, true); create_dataset(name, h5_datatype<T>(), 0, chunk_size_, compression_filter_, true);
} }
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT); h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
hsize_t mem_size = data.size(); hsize_t mem_size = data.size();
hid_t memspace = H5Screate_simple(1, &mem_size, nullptr); h5_handle memspace{H5Screate_simple(1, &mem_size, nullptr), H5Sclose};
int size;
int size = H5Sget_simple_extent_npoints(dataspace); herr_t status;
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"}; { // limit the scope of the dataspace handle
h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
herr_t status = H5Sclose(dataspace);
if(status < 0) size = H5Sget_simple_extent_npoints(*dataspace);
throw iodump_exception{"H5Sclose"}; if(size < 0) {
throw iodump_exception{"H5Sget_simple_extent_npoints"};
hsize_t new_size = size + data.size(); }
status = H5Dset_extent(dataset, &new_size);
if(status < 0)
throw iodump_exception{"H5Pset_extent"}; if(data.size() > 0) {
hsize_t new_size = size + data.size();
status = H5Dset_extent(*dataset, &new_size);
if(status < 0) {
throw iodump_exception{"H5Pset_extent"};
}
}
} // because it will be reopened after this
dataspace = H5Dget_space(dataset); h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
// select the hyperslab of the extended area // select the hyperslab of the extended area
hsize_t pos = size; hsize_t pos = size;
hsize_t extent = data.size(); hsize_t extent = data.size();
status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &pos, nullptr, &extent, nullptr); status = H5Sselect_hyperslab(*dataspace, H5S_SELECT_SET, &pos, nullptr, &extent, nullptr);
if(status < 0) if(status < 0)
throw iodump_exception{"H5Sselect_hyperslap"}; throw iodump_exception{"H5Sselect_hyperslap"};
status = H5Dwrite(dataset, h5_datatype<T>(), memspace, dataspace, H5P_DEFAULT, data.data()); status = H5Dwrite(*dataset, h5_datatype<T>(), *memspace, *dataspace, H5P_DEFAULT, data.data());
if(status < 0) if(status < 0)
throw iodump_exception{"H5Dwrite"}; throw iodump_exception{"H5Dwrite"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Sclose(memspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
} }
template<class T> template<class T>
void iodump::group::read(const std::string& name, std::vector<T>& data) const { void iodump::group::read(const std::string& name, std::vector<T>& data) const {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT); h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
if(dataset < 0) h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace); // rank > 1 will get flattened when loaded. int size = H5Sget_simple_extent_npoints(*dataspace); // rank > 1 will get flattened when loaded.
if(size < 0) if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"}; throw iodump_exception{"H5Sget_simple_extent_npoints"};
data.resize(size); data.resize(size);
herr_t status = H5Dread(dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data()); herr_t status = H5Dread(*dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data());
if(status < 0) if(status < 0)
throw iodump_exception{"H5Dread"}; throw iodump_exception{"H5Dread"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
} }
templat