Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Open sidebar
monticore
EmbeddedMontiArc
generators
CNNArch2Gluon
Commits
68c9be95
Commit
68c9be95
authored
Jan 03, 2020
by
Julian Dierkes
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
added preprocessing to CNNSupervisedTrainer and fixed tests
parent
252d858e
Pipeline
#223970
failed with stages
in 50 seconds
Changes
26
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
26 changed files
with
4574 additions
and
542 deletions
+4574
-542
src/main/resources/templates/gluon/CNNDataLoader.ftl
src/main/resources/templates/gluon/CNNDataLoader.ftl
+7
-43
src/main/resources/templates/gluon/CNNGanTrainer.ftl
src/main/resources/templates/gluon/CNNGanTrainer.ftl
+1
-11
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
+8
-3
src/main/resources/templates/gluon/CNNTrainer.ftl
src/main/resources/templates/gluon/CNNTrainer.ftl
+3
-0
src/test/resources/target_code/CNNDataLoader_Alexnet.py
src/test/resources/target_code/CNNDataLoader_Alexnet.py
+7
-43
src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py
...urces/target_code/CNNDataLoader_CifarClassifierNetwork.py
+7
-43
src/test/resources/target_code/CNNDataLoader_Invariant.py
src/test/resources/target_code/CNNDataLoader_Invariant.py
+7
-43
src/test/resources/target_code/CNNDataLoader_MultipleStreams.py
...st/resources/target_code/CNNDataLoader_MultipleStreams.py
+7
-43
src/test/resources/target_code/CNNDataLoader_RNNencdec.py
src/test/resources/target_code/CNNDataLoader_RNNencdec.py
+7
-43
src/test/resources/target_code/CNNDataLoader_RNNsearch.py
src/test/resources/target_code/CNNDataLoader_RNNsearch.py
+7
-43
src/test/resources/target_code/CNNDataLoader_RNNtest.py
src/test/resources/target_code/CNNDataLoader_RNNtest.py
+7
-43
src/test/resources/target_code/CNNDataLoader_ResNeXt50.py
src/test/resources/target_code/CNNDataLoader_ResNeXt50.py
+7
-43
src/test/resources/target_code/CNNDataLoader_Show_attend_tell.py
...t/resources/target_code/CNNDataLoader_Show_attend_tell.py
+7
-43
src/test/resources/target_code/CNNDataLoader_ThreeInputCNN_M14.py
.../resources/target_code/CNNDataLoader_ThreeInputCNN_M14.py
+7
-43
src/test/resources/target_code/CNNDataLoader_VGG16.py
src/test/resources/target_code/CNNDataLoader_VGG16.py
+7
-43
src/test/resources/target_code/CNNSupervisedTrainer_Alexnet.py
...est/resources/target_code/CNNSupervisedTrainer_Alexnet.py
+9
-4
src/test/resources/target_code/CNNSupervisedTrainer_CifarClassifierNetwork.py
...arget_code/CNNSupervisedTrainer_CifarClassifierNetwork.py
+9
-4
src/test/resources/target_code/CNNSupervisedTrainer_Invariant.py
...t/resources/target_code/CNNSupervisedTrainer_Invariant.py
+519
-0
src/test/resources/target_code/CNNSupervisedTrainer_MultipleStreams.py
...urces/target_code/CNNSupervisedTrainer_MultipleStreams.py
+507
-0
src/test/resources/target_code/CNNSupervisedTrainer_RNNencdec.py
...t/resources/target_code/CNNSupervisedTrainer_RNNencdec.py
+621
-0
src/test/resources/target_code/CNNSupervisedTrainer_RNNsearch.py
...t/resources/target_code/CNNSupervisedTrainer_RNNsearch.py
+618
-0
src/test/resources/target_code/CNNSupervisedTrainer_RNNtest.py
...est/resources/target_code/CNNSupervisedTrainer_RNNtest.py
+587
-0
src/test/resources/target_code/CNNSupervisedTrainer_ResNeXt50.py
...t/resources/target_code/CNNSupervisedTrainer_ResNeXt50.py
+492
-0
src/test/resources/target_code/CNNSupervisedTrainer_Show_attend_tell.py
...rces/target_code/CNNSupervisedTrainer_Show_attend_tell.py
+609
-0
src/test/resources/target_code/CNNSupervisedTrainer_ThreeInputCNN_M14.py
...ces/target_code/CNNSupervisedTrainer_ThreeInputCNN_M14.py
+498
-0
src/test/resources/target_code/CNNSupervisedTrainer_VGG16.py
src/test/resources/target_code/CNNSupervisedTrainer_VGG16.py
+9
-4
No files found.
src/main/resources/templates/gluon/CNNDataLoader.ftl
View file @
68c9be95
...
...
@@ -65,48 +65,6 @@ class ${tc.fileNameWithoutEnding}:
return train_iter, test_iter, data_mean, data_std, train_images, test_images
def load_data_img(self, batch_size, img_size):
train_h5, test_h5 = self.load_h5_files()
width = img_size[0]
height = img_size[1]
comb_data = {}
data_mean = {}
data_std = {}
for input_name in self._input_names_:
train_data = train_h5[input_name][:]
test_data = test_h5[input_name][:]
train_shape = train_data.shape
test_shape = test_data.shape
comb_data[input_name] = mx.nd.zeros((train_shape[0]+test_shape[0], train_shape[1], width, height))
for i, img in enumerate(train_data):
img = img.transpose(1,2,0)
comb_data[input_name][i] = cv2.resize(img, (width, height)).reshape((train_shape[1],width,height))
for i, img in enumerate(test_data):
img = img.transpose(1, 2, 0)
comb_data[input_name][i+train_shape[0]] = cv2.resize(img, (width, height)).reshape((train_shape[1], width, height))
data_mean[input_name + '_'] = nd.array(comb_data[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(comb_data[input_name][:].asnumpy().std(axis=0) + 1e-5)
comb_label = {}
for output_name in self._output_names_:
train_labels = train_h5[output_name][:]
test_labels = test_h5[output_name][:]
comb_label[output_name] = np.append(train_labels, test_labels, axis=0)
train_iter = mx.io.NDArrayIter(data=comb_data,
label=comb_label,
batch_size=batch_size)
test_iter = None
return train_iter, test_iter, data_mean, data_std
def load_preprocessed_data(self, batch_size, preproc_lib):
train_h5, test_h5 = self.load_h5_files()
...
...
@@ -148,6 +106,9 @@ class ${tc.fileNameWithoutEnding}:
data_mean[input_name + '_'] = nd.array(train_data[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_data[input_name][:].asnumpy().std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=batch_size)
...
...
@@ -178,11 +139,14 @@ class ${tc.fileNameWithoutEnding}:
for output_name in self._output_names_:
test_label[output_name][i] = getattr(shape_output, output_name + "_out")
if 'images' in test_h5:
test_images = test_h5['images']
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=batch_size)
return train_iter, test_iter, data_mean, data_std
return train_iter, test_iter, data_mean, data_std
, train_images, test_images
def preprocess_data(self, instance_wrapper, input_wrapper, index, data_h5):
for input_name in self._input_names_:
...
...
src/main/resources/templates/gluon/CNNGanTrainer.ftl
View file @
68c9be95
...
...
@@ -114,7 +114,7 @@ class ${tc.fileNameWithoutEnding}:
# if preprocessing:
# train_iter, test_iter, data_mean, data_std = self._data_loader.load_preprocessed_data(batch_size, preproc_lib)
# else:
# train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size
, img_resize
)
# train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
...
...
@@ -166,16 +166,6 @@ class ${tc.fileNameWithoutEnding}:
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
activation_name = 'sigmoid'
<<<<<<< HEAD
=======
<#list tc.architecture.streams as stream>
input_shape = <#list tc.getStreamInputDimensions(stream) as dimensions>${tc.join(dimensions, ",")}</#list>
</#list>
shape_list = list(input_shape)
shape_list[0] = batch_size
input_shape = tuple(shape_list)
>>>>>>> cdd27689fde2eae0cb6e022d18400f7a696b2f58
metric_dis = mx.metric.create(eval_metric)
metric_gen = mx.metric.create(eval_metric)
<#include "gan/InputGenerator.ftl">
...
...
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
View file @
68c9be95
...
...
@@ -192,7 +192,8 @@ class ${tc.fileNameWithoutEnding}:
context='gpu',
save_attention_image=False,
use_teacher_forcing=False,
normalize=True):
normalize=True,
preprocessing = False):
if context == 'gpu':
mx_context = mx.gpu()
elif context == 'cpu':
...
...
@@ -200,6 +201,12 @@ class ${tc.fileNameWithoutEnding}:
else:
logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
if preprocessing:
preproc_lib = "CNNPreprocessor_${tc.fileNameWithoutEnding?keep_after("CNNSupervisedTrainer_")}_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_preprocessed_data(batch_size, preproc_lib)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size)
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
del optimizer_params['weight_decay']
...
...
@@ -215,8 +222,6 @@ class ${tc.fileNameWithoutEnding}:
del optimizer_params['step_size']
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
else:
...
...
src/main/resources/templates/gluon/CNNTrainer.ftl
View file @
68c9be95
...
...
@@ -43,6 +43,9 @@ if __name__ == "__main__":
<#if (config.normalize)??>
normalize=${config.normalize?string("True","False")},
</#if>
<#if (config.preprocessingName)??>
preprocessing=${config.preprocessingName???string("True","False")},
</#if>
<#if (config.useTeacherForcing)??>
use_teacher_forcing='${config.useTeacherForcing?string("True","False")}',
</#if>
...
...
src/test/resources/target_code/CNNDataLoader_Alexnet.py
View file @
68c9be95
...
...
@@ -64,48 +64,6 @@ class CNNDataLoader_Alexnet:
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
load_data_img
(
self
,
batch_size
,
img_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
width
=
img_size
[
0
]
height
=
img_size
[
1
]
comb_data
=
{}
data_mean
=
{}
data_std
=
{}
for
input_name
in
self
.
_input_names_
:
train_data
=
train_h5
[
input_name
][:]
test_data
=
test_h5
[
input_name
][:]
train_shape
=
train_data
.
shape
test_shape
=
test_data
.
shape
comb_data
[
input_name
]
=
mx
.
nd
.
zeros
((
train_shape
[
0
]
+
test_shape
[
0
],
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
train_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
test_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
+
train_shape
[
0
]]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
comb_label
=
{}
for
output_name
in
self
.
_output_names_
:
train_labels
=
train_h5
[
output_name
][:]
test_labels
=
test_h5
[
output_name
][:]
comb_label
[
output_name
]
=
np
.
append
(
train_labels
,
test_labels
,
axis
=
0
)
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
comb_data
,
label
=
comb_label
,
batch_size
=
batch_size
)
test_iter
=
None
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_preprocessed_data
(
self
,
batch_size
,
preproc_lib
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
...
...
@@ -147,6 +105,9 @@ class CNNDataLoader_Alexnet:
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
if
'images'
in
train_h5
:
train_images
=
train_h5
[
'images'
]
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
train_data
,
label
=
train_label
,
batch_size
=
batch_size
)
...
...
@@ -177,11 +138,14 @@ class CNNDataLoader_Alexnet:
for
output_name
in
self
.
_output_names_
:
test_label
[
output_name
][
i
]
=
getattr
(
shape_output
,
output_name
+
"_out"
)
if
'images'
in
test_h5
:
test_images
=
test_h5
[
'images'
]
test_iter
=
mx
.
io
.
NDArrayIter
(
data
=
test_data
,
label
=
test_label
,
batch_size
=
batch_size
)
return
train_iter
,
test_iter
,
data_mean
,
data_std
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
preprocess_data
(
self
,
instance_wrapper
,
input_wrapper
,
index
,
data_h5
):
for
input_name
in
self
.
_input_names_
:
...
...
src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py
View file @
68c9be95
...
...
@@ -64,48 +64,6 @@ class CNNDataLoader_CifarClassifierNetwork:
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
load_data_img
(
self
,
batch_size
,
img_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
width
=
img_size
[
0
]
height
=
img_size
[
1
]
comb_data
=
{}
data_mean
=
{}
data_std
=
{}
for
input_name
in
self
.
_input_names_
:
train_data
=
train_h5
[
input_name
][:]
test_data
=
test_h5
[
input_name
][:]
train_shape
=
train_data
.
shape
test_shape
=
test_data
.
shape
comb_data
[
input_name
]
=
mx
.
nd
.
zeros
((
train_shape
[
0
]
+
test_shape
[
0
],
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
train_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
test_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
+
train_shape
[
0
]]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
comb_label
=
{}
for
output_name
in
self
.
_output_names_
:
train_labels
=
train_h5
[
output_name
][:]
test_labels
=
test_h5
[
output_name
][:]
comb_label
[
output_name
]
=
np
.
append
(
train_labels
,
test_labels
,
axis
=
0
)
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
comb_data
,
label
=
comb_label
,
batch_size
=
batch_size
)
test_iter
=
None
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_preprocessed_data
(
self
,
batch_size
,
preproc_lib
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
...
...
@@ -147,6 +105,9 @@ class CNNDataLoader_CifarClassifierNetwork:
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
if
'images'
in
train_h5
:
train_images
=
train_h5
[
'images'
]
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
train_data
,
label
=
train_label
,
batch_size
=
batch_size
)
...
...
@@ -177,11 +138,14 @@ class CNNDataLoader_CifarClassifierNetwork:
for
output_name
in
self
.
_output_names_
:
test_label
[
output_name
][
i
]
=
getattr
(
shape_output
,
output_name
+
"_out"
)
if
'images'
in
test_h5
:
test_images
=
test_h5
[
'images'
]
test_iter
=
mx
.
io
.
NDArrayIter
(
data
=
test_data
,
label
=
test_label
,
batch_size
=
batch_size
)
return
train_iter
,
test_iter
,
data_mean
,
data_std
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
preprocess_data
(
self
,
instance_wrapper
,
input_wrapper
,
index
,
data_h5
):
for
input_name
in
self
.
_input_names_
:
...
...
src/test/resources/target_code/CNNDataLoader_Invariant.py
View file @
68c9be95
...
...
@@ -64,48 +64,6 @@ class CNNDataLoader_Invariant:
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
load_data_img
(
self
,
batch_size
,
img_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
width
=
img_size
[
0
]
height
=
img_size
[
1
]
comb_data
=
{}
data_mean
=
{}
data_std
=
{}
for
input_name
in
self
.
_input_names_
:
train_data
=
train_h5
[
input_name
][:]
test_data
=
test_h5
[
input_name
][:]
train_shape
=
train_data
.
shape
test_shape
=
test_data
.
shape
comb_data
[
input_name
]
=
mx
.
nd
.
zeros
((
train_shape
[
0
]
+
test_shape
[
0
],
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
train_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
test_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
+
train_shape
[
0
]]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
comb_label
=
{}
for
output_name
in
self
.
_output_names_
:
train_labels
=
train_h5
[
output_name
][:]
test_labels
=
test_h5
[
output_name
][:]
comb_label
[
output_name
]
=
np
.
append
(
train_labels
,
test_labels
,
axis
=
0
)
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
comb_data
,
label
=
comb_label
,
batch_size
=
batch_size
)
test_iter
=
None
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_preprocessed_data
(
self
,
batch_size
,
preproc_lib
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
...
...
@@ -147,6 +105,9 @@ class CNNDataLoader_Invariant:
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
if
'images'
in
train_h5
:
train_images
=
train_h5
[
'images'
]
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
train_data
,
label
=
train_label
,
batch_size
=
batch_size
)
...
...
@@ -177,11 +138,14 @@ class CNNDataLoader_Invariant:
for
output_name
in
self
.
_output_names_
:
test_label
[
output_name
][
i
]
=
getattr
(
shape_output
,
output_name
+
"_out"
)
if
'images'
in
test_h5
:
test_images
=
test_h5
[
'images'
]
test_iter
=
mx
.
io
.
NDArrayIter
(
data
=
test_data
,
label
=
test_label
,
batch_size
=
batch_size
)
return
train_iter
,
test_iter
,
data_mean
,
data_std
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
preprocess_data
(
self
,
instance_wrapper
,
input_wrapper
,
index
,
data_h5
):
for
input_name
in
self
.
_input_names_
:
...
...
src/test/resources/target_code/CNNDataLoader_MultipleStreams.py
View file @
68c9be95
...
...
@@ -64,48 +64,6 @@ class CNNDataLoader_MultipleStreams:
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
load_data_img
(
self
,
batch_size
,
img_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
width
=
img_size
[
0
]
height
=
img_size
[
1
]
comb_data
=
{}
data_mean
=
{}
data_std
=
{}
for
input_name
in
self
.
_input_names_
:
train_data
=
train_h5
[
input_name
][:]
test_data
=
test_h5
[
input_name
][:]
train_shape
=
train_data
.
shape
test_shape
=
test_data
.
shape
comb_data
[
input_name
]
=
mx
.
nd
.
zeros
((
train_shape
[
0
]
+
test_shape
[
0
],
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
train_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
test_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
+
train_shape
[
0
]]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
comb_label
=
{}
for
output_name
in
self
.
_output_names_
:
train_labels
=
train_h5
[
output_name
][:]
test_labels
=
test_h5
[
output_name
][:]
comb_label
[
output_name
]
=
np
.
append
(
train_labels
,
test_labels
,
axis
=
0
)
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
comb_data
,
label
=
comb_label
,
batch_size
=
batch_size
)
test_iter
=
None
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_preprocessed_data
(
self
,
batch_size
,
preproc_lib
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
...
...
@@ -147,6 +105,9 @@ class CNNDataLoader_MultipleStreams:
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
if
'images'
in
train_h5
:
train_images
=
train_h5
[
'images'
]
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
train_data
,
label
=
train_label
,
batch_size
=
batch_size
)
...
...
@@ -177,11 +138,14 @@ class CNNDataLoader_MultipleStreams:
for
output_name
in
self
.
_output_names_
:
test_label
[
output_name
][
i
]
=
getattr
(
shape_output
,
output_name
+
"_out"
)
if
'images'
in
test_h5
:
test_images
=
test_h5
[
'images'
]
test_iter
=
mx
.
io
.
NDArrayIter
(
data
=
test_data
,
label
=
test_label
,
batch_size
=
batch_size
)
return
train_iter
,
test_iter
,
data_mean
,
data_std
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
preprocess_data
(
self
,
instance_wrapper
,
input_wrapper
,
index
,
data_h5
):
for
input_name
in
self
.
_input_names_
:
...
...
src/test/resources/target_code/CNNDataLoader_RNNencdec.py
View file @
68c9be95
...
...
@@ -64,48 +64,6 @@ class CNNDataLoader_RNNencdec:
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
load_data_img
(
self
,
batch_size
,
img_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
width
=
img_size
[
0
]
height
=
img_size
[
1
]
comb_data
=
{}
data_mean
=
{}
data_std
=
{}
for
input_name
in
self
.
_input_names_
:
train_data
=
train_h5
[
input_name
][:]
test_data
=
test_h5
[
input_name
][:]
train_shape
=
train_data
.
shape
test_shape
=
test_data
.
shape
comb_data
[
input_name
]
=
mx
.
nd
.
zeros
((
train_shape
[
0
]
+
test_shape
[
0
],
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
train_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
test_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
+
train_shape
[
0
]]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
comb_label
=
{}
for
output_name
in
self
.
_output_names_
:
train_labels
=
train_h5
[
output_name
][:]
test_labels
=
test_h5
[
output_name
][:]
comb_label
[
output_name
]
=
np
.
append
(
train_labels
,
test_labels
,
axis
=
0
)
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
comb_data
,
label
=
comb_label
,
batch_size
=
batch_size
)
test_iter
=
None
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_preprocessed_data
(
self
,
batch_size
,
preproc_lib
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
...
...
@@ -147,6 +105,9 @@ class CNNDataLoader_RNNencdec:
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
train_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
if
'images'
in
train_h5
:
train_images
=
train_h5
[
'images'
]
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
train_data
,
label
=
train_label
,
batch_size
=
batch_size
)
...
...
@@ -177,11 +138,14 @@ class CNNDataLoader_RNNencdec:
for
output_name
in
self
.
_output_names_
:
test_label
[
output_name
][
i
]
=
getattr
(
shape_output
,
output_name
+
"_out"
)
if
'images'
in
test_h5
:
test_images
=
test_h5
[
'images'
]
test_iter
=
mx
.
io
.
NDArrayIter
(
data
=
test_data
,
label
=
test_label
,
batch_size
=
batch_size
)
return
train_iter
,
test_iter
,
data_mean
,
data_std
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
preprocess_data
(
self
,
instance_wrapper
,
input_wrapper
,
index
,
data_h5
):
for
input_name
in
self
.
_input_names_
:
...
...
src/test/resources/target_code/CNNDataLoader_RNNsearch.py
View file @
68c9be95
...
...
@@ -64,48 +64,6 @@ class CNNDataLoader_RNNsearch:
return
train_iter
,
test_iter
,
data_mean
,
data_std
,
train_images
,
test_images
def
load_data_img
(
self
,
batch_size
,
img_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
width
=
img_size
[
0
]
height
=
img_size
[
1
]
comb_data
=
{}
data_mean
=
{}
data_std
=
{}
for
input_name
in
self
.
_input_names_
:
train_data
=
train_h5
[
input_name
][:]
test_data
=
test_h5
[
input_name
][:]
train_shape
=
train_data
.
shape
test_shape
=
test_data
.
shape
comb_data
[
input_name
]
=
mx
.
nd
.
zeros
((
train_shape
[
0
]
+
test_shape
[
0
],
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
train_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
for
i
,
img
in
enumerate
(
test_data
):
img
=
img
.
transpose
(
1
,
2
,
0
)
comb_data
[
input_name
][
i
+
train_shape
[
0
]]
=
cv2
.
resize
(
img
,
(
width
,
height
)).
reshape
((
train_shape
[
1
],
width
,
height
))
data_mean
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
mean
(
axis
=
0
))
data_std
[
input_name
+
'_'
]
=
nd
.
array
(
comb_data
[
input_name
][:].
asnumpy
().
std
(
axis
=
0
)
+
1e-5
)
comb_label
=
{}
for
output_name
in
self
.
_output_names_
:
train_labels
=
train_h5
[
output_name
][:]
test_labels
=
test_h5
[
output_name
][:]
comb_label
[
output_name
]
=
np
.
append
(
train_labels
,
test_labels
,
axis
=
0
)
train_iter
=
mx
.
io
.
NDArrayIter
(
data
=
comb_data
,
label
=
comb_label
,
batch_size
=
batch_size
)
test_iter
=
None
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_preprocessed_data
(
self
,
batch_size
,
preproc_lib
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
...
...
@@ -147,6 +105,9 @@ class CNNDataLoader_RNNsearch: