Commit 9a11259d authored by panicPaul's avatar panicPaul
Browse files

merge commit

parent d68ddd43
......@@ -29,18 +29,20 @@ class ${tc.fileNameWithoutEnding}:
train_data[input_name] = train_h5[input_name]
train_dataset = train_h5[input_name]
train_dataset_shape = train_data[input_name].shape
# slize_size limits the memory cosumption, by only loading slizes of size <500MB into memory
slize_size = min(train_dataset_shape[0] - 1, int(500e6 / (train_h5[input_name][0].size * train_h5[input_name][0].itemsize)))
num_slizes = max(1, int(train_h5[input_name].shape[0] / slize_size))
# slice_size limits the memory consumption, by only loading slices of size <500MB into memory
slice_size = min(train_dataset_shape[0] - 1, int(500e6 / (train_h5[input_name][0].size * \
train_h5[input_name][0].itemsize)))
num_slices = max(1, int(train_h5[input_name].shape[0] / slice_size))
mean = np.zeros(train_dataset_shape[1: ])
std = np.zeros(train_dataset_shape[1: ])
for i in range(int(train_dataset_shape[0] / slize_size)):
mean += train_dataset[i * slize_size: (i + 1) * slize_size].mean(axis=0) / num_slizes
std += train_dataset[i * slize_size: (i + 1) * slize_size].std(axis=0) / num_slizes
if slize_size > train_dataset_shape[0] - 1:
mean += train_dataset[num_slizes * slize_size: ].mean(axis=0) / (slize_size - num_slizes % slize_size)
std += train_dataset[num_slizes * slize_size: ].std(axis=0) / (slize_size - num_slizes % slize_size)
for i in range(int(train_dataset_shape[0] / slice_size)):
mean += train_dataset[i * slice_size: (i + 1) * slice_size].mean(axis=0) / num_slices
std += train_dataset[i * slice_size: (i + 1) * slice_size].std(axis=0) / num_slices
if slice_size > train_dataset_shape[0] - 1:
mean += train_dataset[num_slices * slice_size: ].mean(axis=0) / (slice_size - num_slices % slice_size)
std += train_dataset[num_slices * slice_size: ].std(axis=0) / (slice_size - num_slices % slice_size)
std += 1e-5
data_mean[input_name + '_'] = nd.array(mean)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment