Skip to content

GitLab

Commit 40bd643b authored by Lukas Weber's avatar Lukas Weber
Browse files

template functions need to go into headers after all...

parent 5626729b
Hide whitespace changes
Inline Side-by-side
CMakeLists.txt
View file @ 40bd643b
......@@ -20,7 +20,9 @@ set(SRCs
merger.cpp
mc.cpp
runner_single.cpp
runner_task.cpp
)
add_library(load_leveller ${SRCs})
#target_link_libraries(load_leveller ${MPI_LIBRARIES} ${CONAN_LIBS})
install(FILES load_leveller.h DESTINATION include)
conanfile.txt deleted 100644 → 0
View file @ 5626729b
[requires]
fmt/5.2.1@bincrafters/stable
hdf5/1.10.2-dm2@lweber/mc
[generators]
cmake
[options]
fmt:header_only=True
hdf5:shared=False
dump.cpp
View file @ 40bd643b
#include "dump.h"
#include <typeinfo>
#include <fmt/format.h>
// Maybe I should just use a C++ wrapper for HDF5 to avoid all these terrible error checking ifs but somehow I didn’t.
template<typename T>
static hid_t h5_datatype() {
if(typeid(T) == typeid(int))
return H5T_NATIVE_INT;
if(typeid(T) == typeid(short))
return H5T_NATIVE_SHORT;
if(typeid(T) == typeid(long))
return H5T_NATIVE_LONG;
if(typeid(T) == typeid(long long))
return H5T_NATIVE_LLONG;
if(typeid(T) == typeid(unsigned int))
return H5T_NATIVE_UINT;
if(typeid(T) == typeid(unsigned short))
return H5T_NATIVE_USHORT;
if(typeid(T) == typeid(unsigned long))
return H5T_NATIVE_ULONG;
if(typeid(T) == typeid(unsigned long long))
return H5T_NATIVE_ULLONG;
if(typeid(T) == typeid(float))
return H5T_NATIVE_FLOAT;
if(typeid(T) == typeid(double))
return H5T_NATIVE_DOUBLE;
throw iodump_exception{fmt::format("unsupported datatype: {}",typeid(T).name())};
}
static bool filter_available(H5Z_filter_t filter) {
htri_t avail = H5Zfilter_avail(filter);
......@@ -126,122 +97,6 @@ iodump::~iodump() {
H5Fclose(h5_file_);
}
// chunk_size == 0 means contiguous storage
template<class T>
static hid_t create_dataset(hid_t group, const std::string& name, int size, int chunk_size, H5Z_filter_t compression_filter, bool unlimited) {
herr_t status;
hid_t dataspace;
if(not unlimited)
dataspace = H5Screate_simple(1, {size}, nullptr);
else
dataspace = H5Screate_simple(1, {size}, {H5S_UNLIMITED});
hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
if(chunk_size > 1) { // do not use compression on small datasets
status = H5Pset_chunk(plist, 1, {chunk_size});
if(status < 0)
throw iodump_exception{"H5Pset_chunk"};
if(compression_filter== H5Z_FILTER_DEFLATE) {
status = H5Pset_deflate(plist, 6);
if(status < 0)
throw iodump_exception{"H5Pset_deflate"};
}
}
hid_t dataset = H5Dcreate2(group, name.c_str(), h5_datatype<T>(), dataspace, H5P_DEFAULT, plist, H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dcreate2"};
status = H5Pclose(plist);
if(status < 0)
throw iodump_exception{"H5Pclose"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
return dataset;
}
template<class T>
void iodump::write(const std::string& name, const std::vector<T>& data) {
int chunk_size = 0;
H5Z_filter_t compression_filter= 0;
// no compression and chunking unless dataset is big enough
if(data.size() >= chunk_size_) {
chunk_size = chunk_size_;
compression_filter= compression_filter_;
}
hid_t dataset = create_dataset<T>(group_, name, data.size(), chunk_size, compression_filter, false);
herr_t status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::write(const std::string& name, T value) {
// I hope nobody copies a lot of small values...
write(name, std::vector<T>{value});
}
template <class T>
void iodump::insert_back(const std::string& name, const std::vector<T>& data) {
// If the dataset does not exist, we create a new unlimited one with 0 extent.
htri_t exists = H5Lexists(group_, name.c_str(), H5P_DEFAULT);
if(exists == 0) {
create_dataset<T>(group_, name, 0, chunk_size_, compression_filter_, true);
} else if(exists < 0) {
throw iodump_exception{"H5Lexists"};
}
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace);
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
herr_t status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dset_extent(dataset, size + data.size());
if(status < 0)
throw iodump_exception{"H5Pset_extent"};
dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
// select the hyperslab of the extended area
status = H5Sselect_hyperslap(dataspace, H5S_SELECT_SET, {size}, nullptr, {data.size()}, nullptr);
if(status < 0)
throw iodump_exception{"H5Sselect_hyperslap"};
status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, dataspace, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
void iodump::change_group(const std::string& path) {
hid_t tmp = H5Gopen(group_, path.c_str(), H5P_DEFAULT);
if(tmp < 0)
......@@ -252,38 +107,7 @@ void iodump::change_group(const std::string& path) {
group_ = tmp;
}
template<class T>
void iodump::read(const std::string& name, std::vector<T>& data) {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace); // rank > 1 will get flattened when loaded.
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
data.resize(size);
herr_t status = H5Dread(dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dread"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::read(const std::string& name, T& value) {
std::vector<T> buf;
read(name, buf);
value = buf.at(0);
}
size_t iodump::get_extent(const std::string& name) {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
......
dump.h
View file @ 40bd643b
......@@ -3,6 +3,7 @@
#include <vector>
#include <hdf5.h>
#include <string>
#include <fmt/format.h>
struct iodump_exception : public std::exception {
std::string message;
......@@ -66,6 +67,11 @@ public:
void change_group(const std::string &path); // this works like the cd command
private:
iodump(hid_t h5_file);
template<typename T>
static hid_t h5_datatype();
template<typename T>
static hid_t create_dataset(hid_t group, const std::string& name, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited);
const hid_t h5_file_;
hid_t group_;
......@@ -74,3 +80,183 @@ private:
const H5Z_filter_t compression_filter_ = H5Z_FILTER_DEFLATE;
const size_t chunk_size_ = 1000;
};
template<typename T>
hid_t iodump::h5_datatype() {
if(typeid(T) == typeid(int))
return H5T_NATIVE_INT;
if(typeid(T) == typeid(short))
return H5T_NATIVE_SHORT;
if(typeid(T) == typeid(long))
return H5T_NATIVE_LONG;
if(typeid(T) == typeid(long long))
return H5T_NATIVE_LLONG;
if(typeid(T) == typeid(unsigned int))
return H5T_NATIVE_UINT;
if(typeid(T) == typeid(unsigned short))
return H5T_NATIVE_USHORT;
if(typeid(T) == typeid(unsigned long))
return H5T_NATIVE_ULONG;
if(typeid(T) == typeid(unsigned long long))
return H5T_NATIVE_ULLONG;
if(typeid(T) == typeid(float))
return H5T_NATIVE_FLOAT;
if(typeid(T) == typeid(double))
return H5T_NATIVE_DOUBLE;
throw iodump_exception{fmt::format("unsupported datatype: {}",typeid(T).name())};
}
// chunk_size == 0 means contiguous storage
template<class T>
hid_t iodump::create_dataset(hid_t group, const std::string& name, hsize_t size, hsize_t chunk_size, H5Z_filter_t compression_filter, bool unlimited) {
herr_t status;
hid_t dataspace;
if(not unlimited) {
dataspace = H5Screate_simple(1, &size, nullptr);
} else {
hsize_t maxdim = H5S_UNLIMITED;
dataspace = H5Screate_simple(1, &size, &maxdim);
}
hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
if(chunk_size > 1) { // do not use compression on small datasets
status = H5Pset_chunk(plist, 1, &chunk_size);
if(status < 0)
throw iodump_exception{"H5Pset_chunk"};
if(compression_filter== H5Z_FILTER_DEFLATE) {
status = H5Pset_deflate(plist, 6);
if(status < 0)
throw iodump_exception{"H5Pset_deflate"};
}
}
hid_t dataset = H5Dcreate2(group, name.c_str(), h5_datatype<T>(), dataspace, H5P_DEFAULT, plist, H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dcreate2"};
status = H5Pclose(plist);
if(status < 0)
throw iodump_exception{"H5Pclose"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
return dataset;
}
template<class T>
void iodump::write(const std::string& name, const std::vector<T>& data) {
int chunk_size = 0;
H5Z_filter_t compression_filter= 0;
// no compression and chunking unless dataset is big enough
if(data.size() >= chunk_size_) {
chunk_size = chunk_size_;
compression_filter= compression_filter_;
}
hid_t dataset = create_dataset<T>(group_, name, data.size(), chunk_size, compression_filter, false);
herr_t status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::write(const std::string& name, T value) {
// I hope nobody copies a lot of small values...
write(name, std::vector<T>{value});
}
template <class T>
void iodump::insert_back(const std::string& name, const std::vector<T>& data) {
// If the dataset does not exist, we create a new unlimited one with 0 extent.
htri_t exists = H5Lexists(group_, name.c_str(), H5P_DEFAULT);
if(exists == 0) {
create_dataset<T>(group_, name, 0, chunk_size_, compression_filter_, true);
} else if(exists < 0) {
throw iodump_exception{"H5Lexists"};
}
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace);
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
herr_t status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
hsize_t new_size = size + data.size();
status = H5Dset_extent(dataset, &new_size);
if(status < 0)
throw iodump_exception{"H5Pset_extent"};
dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
// select the hyperslab of the extended area
hsize_t pos = size;
hsize_t extent = data.size();
status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &pos, nullptr, &extent, nullptr);
if(status < 0)
throw iodump_exception{"H5Sselect_hyperslap"};
status = H5Dwrite(dataset, h5_datatype<T>(), H5S_ALL, dataspace, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dwrite"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::read(const std::string& name, std::vector<T>& data) {
hid_t dataset = H5Dopen2(group_, name.c_str(), H5P_DEFAULT);
if(dataset < 0)
throw iodump_exception{"H5Dopen2"};
hid_t dataspace = H5Dget_space(dataset);
if(dataspace < 0)
throw iodump_exception{"H5Dget_space"};
int size = H5Sget_simple_extent_npoints(dataspace); // rank > 1 will get flattened when loaded.
if(size < 0)
throw iodump_exception{"H5Sget_simple_extent_npoints"};
data.resize(size);
herr_t status = H5Dread(dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data());
if(status < 0)
throw iodump_exception{"H5Dread"};
status = H5Sclose(dataspace);
if(status < 0)
throw iodump_exception{"H5Sclose"};
status = H5Dclose(dataset);
if(status < 0)
throw iodump_exception{"H5Dclose"};
}
template<class T>
void iodump::read(const std::string& name, T& value) {
std::vector<T> buf;
read(name, buf);
value = buf.at(0);
}
measurements.cpp
View file @ 40bd643b
......@@ -17,12 +17,6 @@ void measurements::add_observable(const std::string& name, size_t bin_size, size
observables_.emplace(name, observable{name, bin_size, vector_length, initial_length});
}
template <class T>
void measurements::add(const std::string name, T value) {
observables_.at(name).add(value);
}
void measurements::checkpoint_write(iodump& dump_file) {
dump_file.change_group("measurements");
......
measurements.h
View file @ 40bd643b
......@@ -29,3 +29,8 @@ public:
private:
std::map<std::string, observable> observables_;
};
template <class T>
void measurements::add(const std::string name, T value) {
observables_.at(name).add(value);
}
observable.cpp
View file @ 40bd643b
......@@ -12,41 +12,10 @@ observable::observable(std::string name, size_t vector_length, size_t bin_length
samples_.reserve(vector_length_*initial_length_);
}
template <class T>
void observable::add(T val) {
std::vector<T> v = {val};
add(v);
}
const std::string& observable::name() const {
return name_;
}
template <class T>
void observable::add(const std::vector<T>& val) {
// handle wrong vector length gracefully on first add
if(current_bin_ == 0 and current_bin_filling_ == 0) {
vector_length_=val.size();
samples_.resize((current_bin_+1)*vector_length_,0.);
} else if(vector_length_ != val.size()) {
throw std::runtime_error{fmt::format("observable::add: added vector has different size ({}) than what was added before ({})", val.size(), vector_length_)};
}
for(size_t j=0; j < vector_length_; ++j)
samples_[j+current_bin_*vector_length_] += static_cast<double>(val[j]);
current_bin_filling_++;
if(current_bin_filling_ == bin_length_) { //need to start a new bin next time
if (bin_length_>1)
for(size_t j = 0; j < vector_length_; ++j)
samples_[current_bin_*vector_length_+j] /= bin_length_;
current_bin_++;
samples_.resize((current_bin_+1)*vector_length_);
current_bin_filling_=0;
}
}
void observable::checkpoint_write(iodump& dump_file) const {
// The plan is that before checkpointing, all complete bins are written to the measurement file.
// Then only the incomplete bin remains and we write that into the dump to resume
......
observable.h
View file @ 40bd643b
#ifndef MCL_OBSERVABLE_H
#define MCL_OBSERVABLE_H
#pragma once
#include <cmath>
#include <string>
......@@ -37,6 +36,32 @@ private:
};
template <class T>
void observable::add(T val) {
std::vector<T> v = {val};
add(v);
}
#endif
template <class T>
void observable::add(const std::vector<T>& val) {
// handle wrong vector length gracefully on first add
if(current_bin_ == 0 and current_bin_filling_ == 0) {
vector_length_=val.size();
samples_.resize((current_bin_+1)*vector_length_,0.);
} else if(vector_length_ != val.size()) {
throw std::runtime_error{fmt::format("observable::add: added vector has different size ({}) than what was added before ({})", val.size(), vector_length_)};
}
for(size_t j=0; j < vector_length_; ++j)
samples_[j+current_bin_*vector_length_] += static_cast<double>(val[j]);
current_bin_filling_++;
if(current_bin_filling_ == bin_length_) { //need to start a new bin next time
if (bin_length_>1)
for(size_t j = 0; j < vector_length_; ++j)
samples_[current_bin_*vector_length_+j] /= bin_length_;
current_bin_++;
samples_.resize((current_bin_+1)*vector_length_);
current_bin_filling_=0;
}
}
runner.h
View file @ 40bd643b
......@@ -61,7 +61,6 @@ class runner
public:
int my_rank, world_size;
runner();
~runner();
int start(const std::string& jobfile, double walltime, double checkpointtime, std::function<abstract_mc* (std::string &)> mccreator, int argc, char **argv);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment