Commit 40bd643b authored by Lukas Weber's avatar Lukas Weber
Browse files

template functions need to go into headers after all...

parent 5626729b
......@@ -20,7 +20,9 @@ set(SRCs
merger.cpp
mc.cpp
runner_single.cpp
runner_task.cpp
)
add_library(load_leveller ${SRCs})
#target_link_libraries(load_leveller ${MPI_LIBRARIES} ${CONAN_LIBS})
install(FILES load_leveller.h DESTINATION include)
[requires]
fmt/5.2.1@bincrafters/stable
hdf5/1.10.2-dm2@lweber/mc
[generators]
cmake
[options]
fmt:header_only=True
hdf5:shared=False
#include "dump.h"
#include <typeinfo>
#include <fmt/format.h>
// Maybe I should just use a C++ wrapper for HDF5 to avoid all these terrible error checking ifs but somehow I didn’t.
template<typename T>
static hid_t h5_datatype() {
if(typeid(T) == typeid(int))
return H5T_NATIVE_INT;
if(typeid(T) == typeid(short))
return H5T_NATIVE_SHORT;
if(typeid(T) == typeid(long))
return H5T_NATIVE_LONG;
if(typeid(T) == typeid(long long))
return H5T_NATIVE_LLONG;
if(typeid(T) == typeid(unsigned int))
return H5T_NATIVE_UINT;
if(typeid(T) == typeid(unsigned short))
return H5T_NATIVE_USHORT;
if(typeid(T) == typeid(unsigned long))
return H5T_NATIVE_ULONG;
if(typeid(T) == typeid(unsigned long long))
return H5T_NATIVE_ULLONG;
if(typeid(T) == typeid(float))
return H5T_NATIVE_FLOAT;
if(typeid(T) == typeid(double))
return H5T_NATIVE_DOUBLE;
throw iodump_exception{fmt::format("unsupported datatype: {}",typeid(T).name())};
}
static bool filter_available(H5Z_filter_t filter) {
htri_t avail = H5Zfilter_avail(filter);
......@@ -126,122 +97,6 @@ iodump::~iodump() {
H5Fclose(h5_file_);
}
// chunk_size == 0 means contiguous storage
template<class T>
static hid_t create_dataset(hid_t group, const std::string& name, int size, int chunk_size, H5Z_filter_t compression_filter, bool unlimited) {
herr_t status;
hid_t dataspace;
if(not unlimited)
dataspace = H5Screate_simple(1, {size}, nullptr);
else
dataspace = H5Screate_simple(1, {size}, {H5S_UNLIMITED});
hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
if(chunk_size > 1) { // do not use compression on small datasets
status = H5Pset_chunk(plist, 1, {chunk_size});
if(status < 0)
throw iodump_exception{"H5Pset_chunk"};
if(compression_filter== H5Z_FILTER_DEFLATE) {
status = H5Pset_deflate(plist, 6);
if(status < 0)
throw iodump_exception{"H5Pset_deflate"};
}
}
hid_t dataset = H5Dcreate2(group, name.c_str(), h5_datatype<T>(), dataspace, H5P_DEFAULT, plist, H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dcreate2"};
status = H5Pclose(plist);
if(status < 0)
throw iodump_exception{"H5Pclose"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
return dataset;
}
template<class T>
void iodump::write(const std::string& name, const std::vector<T>& data) {
int chunk_size = 0;
H5Z_filter_t compression_filter= 0;
// no compression and chunking unless dataset is big enough
if(data.size() >= chunk_size_) {
chunk_size = chunk_size_;
compression_filter= compression_filter_;
}
hid_t dataset = create_dataset<T>(group_, name, data.size(), chunk_size, compression_filter, false);
herr_t status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::write(const std::string& name, T value) {
// I hope nobody copies a lot of small values...
write(name, std::vector<T>{value});
}
template <class T>
void iodump::insert_back(const std::string& name, const std::vector<T>& data) {
// If the dataset does not exist, we create a new unlimited one with 0 extent.
htri_t exists = H5Lexists(group_, name.c_str(), H5P_DEFAULT);
if(exists == 0) {
create_dataset<T>(group_, name, 0, chunk_size_, compression_filter_, true);
} else if(exists < 0) {
throw iodump_exception{"H5Lexists"};
}
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace);
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
herr_t status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dset_extent(dataset, size + data.size());
if(status < 0)
throw iodump_exception{"H5Pset_extent"};
dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
// select the hyperslab of the extended area
status = H5Sselect_hyperslap(dataspace, H5S_SELECT_SET, {size}, nullptr, {data.size()}, nullptr);
if(status < 0)
throw iodump_exception{"H5Sselect_hyperslap"};
status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, dataspace, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
void iodump::change_group(const std::string& path) {
hid_t tmp = H5Gopen(group_, path.c_str(), H5P_DEFAULT);
if(tmp < 0)
......@@ -252,38 +107,7 @@ void iodump::change_group(const std::string& path) {
group_ = tmp;
}
template<class T>
void iodump::read(const std::string& name, std::vector<T>& data) {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace); // rank > 1 will get flattened when loaded.
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
data.resize(size);
herr_t status = H5Dread(dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dread"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::read(const std::string& name, T& value) {
std::vector<T> buf;
read(name, buf);
value = buf.at(0);
}
size_t iodump::get_extent(const std::string& name) {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
......
......@@ -3,6 +3,7 @@
#include <vector>
#include <hdf5.h>
#include <string>
#include <fmt/format.h>
struct iodump_exception : public std::exception {
std::string message;
......@@ -66,6 +67,11 @@ public:
void change_group(const std::string &path); // this works like the cd command
private:
iodump(hid_t h5_file);
template<typename T>
static hid_t h5_datatype();
template<typename T>
static hid_t create_dataset(hid_t group, const std::string& name, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited);
const hid_t h5_file_;
hid_t group_;
......@@ -74,3 +80,183 @@ private:
const H5Z_filter_t compression_filter_ = H5Z_FILTER_DEFLATE;
const size_t chunk_size_ = 1000;
};
template<typename T>
hid_t iodump::h5_datatype() {
if(typeid(T) == typeid(int))
return H5T_NATIVE_INT;
if(typeid(T) == typeid(short))
return H5T_NATIVE_SHORT;
if(typeid(T) == typeid(long))
return H5T_NATIVE_LONG;
if(typeid(T) == typeid(long long))
return H5T_NATIVE_LLONG;
if(typeid(T) == typeid(unsigned int))
return H5T_NATIVE_UINT;
if(typeid(T) == typeid(unsigned short))
return H5T_NATIVE_USHORT;
if(typeid(T) == typeid(unsigned long))
return H5T_NATIVE_ULONG;
if(typeid(T) == typeid(unsigned long long))
return H5T_NATIVE_ULLONG;
if(typeid(T) == typeid(float))
return H5T_NATIVE_FLOAT;
if(typeid(T) == typeid(double))
return H5T_NATIVE_DOUBLE;
throw iodump_exception{fmt::format("unsupported datatype: {}",typeid(T).name())};
}
// chunk_size == 0 means contiguous storage
template<class T>
hid_t iodump::create_dataset(hid_t group, const std::string& name, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited) {
herr_t status;
hid_t dataspace;
if(not unlimited) {
dataspace = H5Screate_simple(1, &size, nullptr);
} else {
hsize_t maxdim = H5S_UNLIMITED;
dataspace = H5Screate_simple(1, &size, &maxdim);
}
hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
if(chunk_size > 1) { // do not use compression on small datasets
status = H5Pset_chunk(plist, 1, &chunk_size);
if(status < 0)
throw iodump_exception{"H5Pset_chunk"};
if(compression_filter== H5Z_FILTER_DEFLATE) {
status = H5Pset_deflate(plist, 6);
if(status < 0)
throw iodump_exception{"H5Pset_deflate"};
}
}
hid_t dataset = H5Dcreate2(group, name.c_str(), h5_datatype<T>(), dataspace, H5P_DEFAULT, plist, H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dcreate2"};
status = H5Pclose(plist);
if(status < 0)
throw iodump_exception{"H5Pclose"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
return dataset;
}
template<class T>
void iodump::write(const std::string& name, const std::vector<T>& data) {
int chunk_size = 0;
H5Z_filter_t compression_filter= 0;
// no compression and chunking unless dataset is big enough
if(data.size() >= chunk_size_) {
chunk_size = chunk_size_;
compression_filter= compression_filter_;
}
hid_t dataset = create_dataset<T>(group_, name, data.size(), chunk_size, compression_filter, false);
herr_t status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::write(const std::string& name, T value) {
// I hope nobody copies a lot of small values...
write(name, std::vector<T>{value});
}
template <class T>
void iodump::insert_back(const std::string& name, const std::vector<T>& data) {
// If the dataset does not exist, we create a new unlimited one with 0 extent.
htri_t exists = H5Lexists(group_, name.c_str(), H5P_DEFAULT);
if(exists == 0) {
create_dataset<T>(group_, name, 0, chunk_size_, compression_filter_, true);
} else if(exists < 0) {
throw iodump_exception{"H5Lexists"};
}
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace);
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
herr_t status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
hsize_t new_size = size + data.size();
status = H5Dset_extent(dataset, &new_size);
if(status < 0)
throw iodump_exception{"H5Pset_extent"};
dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
// select the hyperslab of the extended area
hsize_t pos = size;
hsize_t extent = data.size();
status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &pos, nullptr, &extent, nullptr);
if(status < 0)
throw iodump_exception{"H5Sselect_hyperslap"};
status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, dataspace, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::read(const std::string& name, std::vector<T>& data) {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace); // rank > 1 will get flattened when loaded.
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
data.resize(size);
herr_t status = H5Dread(dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dread"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::read(const std::string& name, T& value) {
std::vector<T> buf;
read(name, buf);
value = buf.at(0);
}
......@@ -17,12 +17,6 @@ void measurements::add_observable(const std::string& name, size_t bin_size, size
observables_.emplace(name, observable{name, bin_size, vector_length, initial_length});
}
template <class T>
void measurements::add(const std::string name, T value) {
observables_.at(name).add(value);
}
void measurements::checkpoint_write(iodump& dump_file) {
dump_file.change_group("measurements");
......
......@@ -29,3 +29,8 @@ public:
private:
std::map<std::string, observable> observables_;
};
template <class T>
void measurements::add(const std::string name, T value) {
observables_.at(name).add(value);
}
......@@ -12,41 +12,10 @@ observable::observable(std::string name, size_t vector_length, size_t bin_length
samples_.reserve(vector_length_*initial_length_);
}
template <class T>
void observable::add(T val) {
std::vector<T> v = {val};
add(v);
}
const std::string& observable::name() const {
return name_;
}
template <class T>
void observable::add(const std::vector<T>& val) {
// handle wrong vector length gracefully on first add
if(current_bin_ == 0 and current_bin_filling_ == 0) {
vector_length_=val.size();
samples_.resize((current_bin_+1)*vector_length_,0.);
} else if(vector_length_ != val.size()) {
throw std::runtime_error{fmt::format("observable::add: added vector has different size ({}) than what was added before ({})", val.size(), vector_length_)};
}
for(size_t j=0; j < vector_length_; ++j)
samples_[j+current_bin_*vector_length_] += static_cast<double>(val[j]);
current_bin_filling_++;
if(current_bin_filling_ == bin_length_) { //need to start a new bin next time
if (bin_length_>1)
for(size_t j = 0; j < vector_length_; ++j)
samples_[current_bin_*vector_length_+j] /= bin_length_;
current_bin_++;
samples_.resize((current_bin_+1)*vector_length_);
current_bin_filling_=0;
}
}
void observable::checkpoint_write(iodump& dump_file) const {
// The plan is that before checkpointing, all complete bins are written to the measurement file.
// Then only the incomplete bin remains and we write that into the dump to resume
......
#ifndef MCL_OBSERVABLE_H
#define MCL_OBSERVABLE_H
#pragma once
#include <cmath>
#include <string>
......@@ -37,6 +36,32 @@ private:
};
template <class T>
void observable::add(T val) {
std::vector<T> v = {val};
add(v);
}
#endif
template <class T>
void observable::add(const std::vector<T>& val) {
// handle wrong vector length gracefully on first add
if(current_bin_ == 0 and current_bin_filling_ == 0) {
vector_length_=val.size();
samples_.resize((current_bin_+1)*vector_length_,0.);
} else if(vector_length_ != val.size()) {
throw std::runtime_error{fmt::format("observable::add: added vector has different size ({}) than what was added before ({})", val.size(), vector_length_)};
}
for(size_t j=0; j < vector_length_; ++j)
samples_[j+current_bin_*vector_length_] += static_cast<double>(val[j]);
current_bin_filling_++;
if(current_bin_filling_ == bin_length_) { //need to start a new bin next time
if (bin_length_>1)
for(size_t j = 0; j < vector_length_; ++j)
samples_[current_bin_*vector_length_+j] /= bin_length_;
current_bin_++;
samples_.resize((current_bin_+1)*vector_length_);
current_bin_filling_=0;
}
}
......@@ -61,7 +61,6 @@ class runner
public:
int my_rank, world_size;
runner();
~runner();
int start(const std::string& jobfile, double walltime, double checkpointtime, std::function<abstract_mc* (std::string &)> mccreator, int argc, char **argv);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment