data_to_single_hdf5.py 1.08 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
import datetime
import h5py
import os

HDF5_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_HDF5/"


start_date = datetime.datetime.now()
big_file = h5py.File(HDF5_PATH + "all_train.h5")
dset_image = big_file.create_dataset("image", (0, 3, 210, 280), maxshape=(None, 3,210,280),  chunks=True)
dset_labels = big_file.create_dataset("predictions_label", (0, 13), maxshape=(None, 13), chunks=True)

data_start_idx = 0
for file_num in range(1, 485):
    file_path = HDF5_PATH + "train_" + str(file_num) + ".h5"
    print(file_path)
    with h5py.File(file_path, "r") as f:
        images = f["image"]
        labels = f["predictions_label"]

        dset_image.resize( (dset_image.shape[0]+1000, 3,210,280) )
        dset_image[data_start_idx:data_start_idx+1000,:,:,:] = images

        dset_labels.resize( (dset_labels.shape[0]+1000, 13) )
        dset_labels[data_start_idx:data_start_idx+1000,:] = labels

        data_start_idx += 1000
    os.remove(file_path)

    end_time = datetime.datetime.now()
    elapsed_time = end_time - start_date
    print("Elapsed time " + str(elapsed_time))