dump.h 7.8 KB
Newer Older
1
#pragma once
Stefan Weßel's avatar
ic  
Stefan Weßel committed
2

3
#include <fmt/format.h>
4
#include <hdf5.h>
Lukas Weber's avatar
Lukas Weber committed
5
#include <string>
6
#include <vector>
Stefan Weßel's avatar
ic  
Stefan Weßel committed
7

8
9
namespace loadl {

10
11
12
class iodump_exception : public std::exception {
private:
	std::string message_;
13

14
public:
15
	iodump_exception(const std::string &msg);
16
17

	const char *what() const noexcept;
Stefan Weßel's avatar
ic  
Stefan Weßel committed
18
19
};

20
21
22
23
// This thing is a wrapper around an HDF5 file which can do both reading and
// writing depending on how you open it. If you write to a read only file,
// there will be an error probably.
class iodump {
24
25
26
27
28
29
private:
	// helper class to make sure those pesky HDF5 hid_t handles are always closed
	class h5_handle {
	public:
		h5_handle(hid_t handle, herr_t (*closer)(hid_t));
		~h5_handle();
30
31
		h5_handle(h5_handle &) = delete;
		h5_handle(h5_handle &&) noexcept;
32
		hid_t operator*();
33

34
35
36
37
	private:
		herr_t (*closer_)(hid_t);
		hid_t handle_;
	};
38

39
40
41
42
43
44
45
46
	iodump(std::string filename, hid_t h5_file);

	const std::string filename_;
	const hid_t h5_file_;

	// TODO: make these variable if necessary
	static const H5Z_filter_t compression_filter_ = H5Z_FILTER_DEFLATE;
	static const size_t chunk_size_ = 1000;
47

48
49
50
	template<typename T>
	constexpr static hid_t h5_datatype();

51
public:
52
53
54
	// Wrapper around the concept of a HDF5 group.
	// You can list over the group elements by using the iterators.
	class group {
55
56
57
58
59
	public:
		struct iterator {
			iterator(hid_t group, uint64_t idx);
			std::string operator*();
			iterator operator++();
60
			bool operator!=(const iterator &b);
61
62
63
64

			const hid_t group_;
			uint64_t idx_;
		};
65
66
67
68

		group(hid_t parent, const std::string &path);
		group(const group &) =
		    delete; // did you know this is a thing? Very handy in preventing errors.
69
70
71
		~group();
		iterator begin() const;
		iterator end() const;
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

		template<class T>
		void write(const std::string &name, const std::vector<T> &data) const;
		template<class T>
		void write(const std::string &name,
		           const T &value) const; // this is meant for atomic datatypes like int and double

		// insert_back inserts data at the end of the dataset given by name, extending it if
		// necessary. This only works in read/write mode.
		template<class T>
		void insert_back(const std::string &name, const std::vector<T> &data) const;

		template<class T>
		void read(const std::string &name, std::vector<T> &data) const;
		template<class T>
		void read(const std::string &name, T &value) const;

		size_t get_extent(const std::string &name) const;
90
91
92

		group open_group(const std::string &path) const; // this works like the cd command

93
94
		bool exists(
		    const std::string &path) const; // checks whether an object in the dump file exists
95
	private:
96
97
98
		hid_t group_;

		// chunk_size == 0 means contiguous storage
99
100
101
102
103
		// if the dataset already exists, we try to overwrite it. However it must have the same
		// extent for that to work.
		iodump::h5_handle create_dataset(const std::string &name, hid_t datatype, hsize_t size,
		                                 hsize_t chunk_size, H5Z_filter_t compression_filter,
		                                 bool unlimited) const;
104
	};
105

106
	// delete what was there and create a new file for writing
107
	static iodump create(const std::string &filename);
108

109
110
	static iodump open_readonly(const std::string &filename);
	static iodump open_readwrite(const std::string &filename);
111

112
	group get_root();
113

114
115
116
	// TODO: once the intel compiler can do guaranteed copy elision,
	// please uncomment this line! and be careful about bugs!
	//iodump(iodump &) = delete;
117
118
	~iodump();

119
	friend class group;
Stefan Weßel's avatar
ic  
Stefan Weßel committed
120
};
121
122

template<typename T>
123
constexpr hid_t iodump::h5_datatype() {
Lukas Weber's avatar
Lukas Weber committed
124
125
	if(typeid(T) == typeid(signed char))
		return H5T_NATIVE_SCHAR;
126
127
128
129
130
131
132
133
	if(typeid(T) == typeid(int))
		return H5T_NATIVE_INT;
	if(typeid(T) == typeid(short))
		return H5T_NATIVE_SHORT;
	if(typeid(T) == typeid(long))
		return H5T_NATIVE_LONG;
	if(typeid(T) == typeid(long long))
		return H5T_NATIVE_LLONG;
Lukas Weber's avatar
Lukas Weber committed
134
135
	if(typeid(T) == typeid(unsigned char))
		return H5T_NATIVE_UCHAR;
136
137
138
139
140
141
142
143
144
145
146
147
148
	if(typeid(T) == typeid(unsigned int))
		return H5T_NATIVE_UINT;
	if(typeid(T) == typeid(unsigned short))
		return H5T_NATIVE_USHORT;
	if(typeid(T) == typeid(unsigned long))
		return H5T_NATIVE_ULONG;
	if(typeid(T) == typeid(unsigned long long))
		return H5T_NATIVE_ULLONG;
	if(typeid(T) == typeid(float))
		return H5T_NATIVE_FLOAT;
	if(typeid(T) == typeid(double))
		return H5T_NATIVE_DOUBLE;

149
	throw std::runtime_error{fmt::format("unsupported datatype: {}", typeid(T).name())};
150
	// If you run into this error, you probably tried to write a non-primitive datatype
Lukas Weber's avatar
Lukas Weber committed
151
152
	// to a dump file. See the other classes’s checkpointing functions for an example of
	// what to do.
153
	// ... or it is a native datatype I forgot to add. Then add it.
154
155
156
}

template<class T>
157
void iodump::group::write(const std::string &name, const std::vector<T> &data) const {
158
	int chunk_size = 0;
159
	H5Z_filter_t compression_filter = 0;
160

161
	// no compression and chunking unless dataset is big enough
162
	if(data.size() >= chunk_size_) {
163
		chunk_size = iodump::chunk_size_;
164
		compression_filter = iodump::compression_filter_;
165
166
	}

167
168
169
170
	h5_handle dataset{
	    create_dataset(name, h5_datatype<T>(), data.size(), chunk_size, compression_filter, false)};
	herr_t status =
	    H5Dwrite(*dataset, h5_datatype<T>(), H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data());
171
172
173
174
175
	if(status < 0)
		throw iodump_exception{"H5Dwrite"};
}

template<class T>
176
void iodump::group::write(const std::string &name, const T &value) const {
177
178
179
180
	// I hope nobody copies a lot of small values...
	write(name, std::vector<T>{value});
}

181
template<>
182
inline void iodump::group::write(const std::string &name, const std::string &value) const {
183
184
185
	write(name, std::vector<char>{value.begin(), value.end()});
}

186
187
template<class T>
void iodump::group::insert_back(const std::string &name, const std::vector<T> &data) const {
188
	// If the dataset does not exist, we create a new unlimited one with 0 extent.
189
190
	if(!exists(name)) {
		create_dataset(name, h5_datatype<T>(), 0, chunk_size_, compression_filter_, true);
191
192
	}

193
	h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
194

195
	hsize_t mem_size = data.size();
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
	h5_handle memspace{H5Screate_simple(1, &mem_size, nullptr), H5Sclose};
	int size;
	herr_t status;

	{ // limit the scope of the dataspace handle
		h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};

		size = H5Sget_simple_extent_npoints(*dataspace);
		if(size < 0) {
			throw iodump_exception{"H5Sget_simple_extent_npoints"};
		}

		if(data.size() > 0) {
			hsize_t new_size = size + data.size();
			status = H5Dset_extent(*dataset, &new_size);
			if(status < 0) {
				throw iodump_exception{"H5Pset_extent"};
			}
		}
	} // because it will be reopened after this
216

217
	h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
218
219
220
221

	// select the hyperslab of the extended area
	hsize_t pos = size;
	hsize_t extent = data.size();
222
	status = H5Sselect_hyperslab(*dataspace, H5S_SELECT_SET, &pos, nullptr, &extent, nullptr);
223
224
225
	if(status < 0)
		throw iodump_exception{"H5Sselect_hyperslap"};

226
	status = H5Dwrite(*dataset, h5_datatype<T>(), *memspace, *dataspace, H5P_DEFAULT, data.data());
227
228
229
230
231
	if(status < 0)
		throw iodump_exception{"H5Dwrite"};
}

template<class T>
232
void iodump::group::read(const std::string &name, std::vector<T> &data) const {
233
234
	h5_handle dataset{H5Dopen2(group_, name.c_str(), H5P_DEFAULT), H5Dclose};
	h5_handle dataspace{H5Dget_space(*dataset), H5Sclose};
235

236
	int size = H5Sget_simple_extent_npoints(*dataspace); // rank > 1 will get flattened when loaded.
237
238
239
	if(size < 0)
		throw iodump_exception{"H5Sget_simple_extent_npoints"};
	data.resize(size);
240
241
242

	herr_t status =
	    H5Dread(*dataset, h5_datatype<T>(), H5S_ALL, H5P_DEFAULT, H5P_DEFAULT, data.data());
243
244
245
246
	if(status < 0)
		throw iodump_exception{"H5Dread"};
}

247
template<>
248
inline void iodump::group::read(const std::string &name, std::string &value) const {
249
250
251
252
253
	std::vector<char> buf;
	read(name, buf);
	value = std::string{buf.begin(), buf.end()};
}

254
template<class T>
255
void iodump::group::read(const std::string &name, T &value) const {
256
257
258
259
	std::vector<T> buf;
	read(name, buf);
	value = buf.at(0);
}
260
}