Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
C
CNNArch2Gluon
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
1
Issues
1
List
Boards
Labels
Service Desk
Milestones
Iterations
Merge Requests
0
Merge Requests
0
Requirements
Requirements
List
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Test Cases
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Package Registry
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issue
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
monticore
EmbeddedMontiArc
generators
CNNArch2Gluon
Commits
2dd5aae6
Commit
2dd5aae6
authored
Apr 08, 2019
by
Evgeny Kusmenko
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'move-generated-train-files' into 'master'
Move generated training files See merge request
!13
parents
af4a9cb2
e5d1c8d6
Pipeline
#125343
canceled with stages
Changes
18
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
564 additions
and
770 deletions
+564
-770
pom.xml
pom.xml
+1
-1
src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java
...e/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java
+3
-0
src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java
.../lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java
+9
-2
src/main/resources/templates/gluon/CNNCreator.ftl
src/main/resources/templates/gluon/CNNCreator.ftl
+0
-189
src/main/resources/templates/gluon/CNNDataLoader.ftl
src/main/resources/templates/gluon/CNNDataLoader.ftl
+57
-0
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
+141
-0
src/main/resources/templates/gluon/CNNTrainer.ftl
src/main/resources/templates/gluon/CNNTrainer.ftl
+8
-2
src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java
.../lang/monticar/cnnarch/gluongenerator/GenerationTest.java
+9
-3
src/test/resources/target_code/CNNCreator_Alexnet.py
src/test/resources/target_code/CNNCreator_Alexnet.py
+0
-189
src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
...esources/target_code/CNNCreator_CifarClassifierNetwork.py
+0
-189
src/test/resources/target_code/CNNCreator_VGG16.py
src/test/resources/target_code/CNNCreator_VGG16.py
+0
-189
src/test/resources/target_code/CNNDataLoader_Alexnet.py
src/test/resources/target_code/CNNDataLoader_Alexnet.py
+57
-0
src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py
...urces/target_code/CNNDataLoader_CifarClassifierNetwork.py
+57
-0
src/test/resources/target_code/CNNDataLoader_VGG16.py
src/test/resources/target_code/CNNDataLoader_VGG16.py
+57
-0
src/test/resources/target_code/CNNTrainer_emptyConfig.py
src/test/resources/target_code/CNNTrainer_emptyConfig.py
+8
-2
src/test/resources/target_code/CNNTrainer_fullConfig.py
src/test/resources/target_code/CNNTrainer_fullConfig.py
+8
-2
src/test/resources/target_code/CNNTrainer_simpleConfig.py
src/test/resources/target_code/CNNTrainer_simpleConfig.py
+8
-2
src/test/resources/target_code/supervised_trainer.py
src/test/resources/target_code/supervised_trainer.py
+141
-0
No files found.
pom.xml
View file @
2dd5aae6
...
...
@@ -8,7 +8,7 @@
<groupId>
de.monticore.lang.monticar
</groupId>
<artifactId>
cnnarch-gluon-generator
</artifactId>
<version>
0.1.
5
</version>
<version>
0.1.
6
</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
...
...
src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java
View file @
2dd5aae6
...
...
@@ -47,6 +47,9 @@ public class CNNArch2Gluon extends CNNArch2MxNet {
temp
=
archTc
.
process
(
"CNNNet"
,
Target
.
PYTHON
);
fileContentMap
.
put
(
temp
.
getKey
(),
temp
.
getValue
());
temp
=
archTc
.
process
(
"CNNDataLoader"
,
Target
.
PYTHON
);
fileContentMap
.
put
(
temp
.
getKey
(),
temp
.
getValue
());
temp
=
archTc
.
process
(
"CNNCreator"
,
Target
.
PYTHON
);
fileContentMap
.
put
(
temp
.
getKey
(),
temp
.
getValue
());
...
...
src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java
View file @
2dd5aae6
...
...
@@ -21,7 +21,14 @@ public class CNNTrain2Gluon extends CNNTrain2MxNet {
configDataList
.
add
(
configData
);
Map
<
String
,
Object
>
ftlContext
=
Collections
.
singletonMap
(
"configurations"
,
configDataList
);
String
templateContent
=
templateConfiguration
.
processTemplate
(
ftlContext
,
"CNNTrainer.ftl"
);
return
Collections
.
singletonMap
(
"CNNTrainer_"
+
getInstanceName
()
+
".py"
,
templateContent
);
Map
<
String
,
String
>
fileContentMap
=
new
HashMap
<>();
String
cnnTrainTemplateContent
=
templateConfiguration
.
processTemplate
(
ftlContext
,
"CNNTrainer.ftl"
);
fileContentMap
.
put
(
"CNNTrainer_"
+
getInstanceName
()
+
".py"
,
cnnTrainTemplateContent
);
String
cnnSupervisedTrainerContent
=
templateConfiguration
.
processTemplate
(
ftlContext
,
"CNNSupervisedTrainer.ftl"
);
fileContentMap
.
put
(
"supervised_trainer.py"
,
cnnSupervisedTrainerContent
);
return
fileContentMap
;
}
}
\ No newline at end of file
src/main/resources/templates/gluon/CNNCreator.ftl
View file @
2dd5aae6
import mxnet as mx
import logging
import os
import errno
import shutil
import h5py
import sys
import numpy as np
import time
from mxnet import gluon, autograd, nd
from CNNNet_${tc.fullArchitectureName} import Net
@mx.init.register
class MyConstant(mx.init.Initializer):
def __init__(self, value):
super(MyConstant, self).__init__(value=value)
self.value = value
def _init_weight(self, _, arr):
arr[:] = mx.nd.array(self.value)
class ${tc.fileNameWithoutEnding}:
_data_dir_ = "${tc.dataPath}/"
_model_dir_ = "model/${tc.componentName}/"
_model_prefix_ = "model"
_input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
_input_shapes_ = [<#list tc.architecture.inputs as input>(${tc.join(input.definition.type.dimensions, ",")})</#list>]
_output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
def __init__(self):
self.weight_initializer = mx.init.Normal()
...
...
@@ -60,176 +41,6 @@ class ${tc.fileNameWithoutEnding}:
return lastEpoch
def load_data(self, batch_size):
train_h5, test_h5 = self.load_h5_files()
data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
train_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
test_iter = None
if test_h5 != None:
test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
test_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
return train_iter, test_iter, data_mean, data_std
def load_h5_files(self):
train_h5 = None
test_h5 = None
train_path = self._data_dir_ + "train.h5"
test_path = self._data_dir_ + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
test_iter = None
if os.path.isfile(test_path):
test_h5 = h5py.File(test_path, 'r')
if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
else:
logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
return train_h5, test_h5
else:
logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
sys.exit(1)
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
optimizer='adam',
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
elif context == 'cpu':
mx_context = mx.cpu()
else:
logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
del optimizer_params['weight_decay']
if 'learning_rate_decay' in optimizer_params:
min_learning_rate = 1e-08
if 'learning_rate_minimum' in optimizer_params:
min_learning_rate = optimizer_params['learning_rate_minimum']
del optimizer_params['learning_rate_minimum']
optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
optimizer_params['step_size'],
factor=optimizer_params['learning_rate_decay'],
stop_factor_lr=min_learning_rate)
del optimizer_params['step_size']
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
if self.net == None:
if normalize:
self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
else:
self.construct(context=mx_context)
begin_epoch = 0
if load_checkpoint:
begin_epoch = self.load(mx_context)
else:
if os.path.isdir(self._model_dir_):
shutil.rmtree(self._model_dir_)
try:
os.makedirs(self._model_dir_)
except OSError:
if not os.path.isdir(self._model_dir_):
raise
trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
if self.net.last_layer == 'softmax':
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
elif self.net.last_layer == 'sigmoid':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif self.net.last_layer == 'linear':
loss_function = mx.gluon.loss.L2Loss()
else: # TODO: Change default?
loss_function = mx.gluon.loss.L2Loss()
logging.warning("Invalid last_layer, defaulting to L2 loss")
speed_period = 50
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
with autograd.record():
output = self.net(data)
loss = loss_function(output, label)
loss.backward()
trainer.step(batch_size)
if tic is None:
tic = time.time()
else:
if batch_i % speed_period == 0:
try:
speed = speed_period * batch_size / (time.time() - tic)
except ZeroDivisionError:
speed = float("inf")
logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
tic = time.time()
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self.net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(test_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self.net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if (epoch - begin_epoch) % checkpoint_period == 0:
self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
+ str(num_epoch + begin_epoch).zfill(4) + '.params')
self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
...
...
src/main/resources/templates/gluon/CNNDataLoader.ftl
0 → 100644
View file @
2dd5aae6
import os
import h5py
import mxnet as mx
import logging
import sys
class ${tc.fullArchitectureName}DataLoader:
_input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
_output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
def __init__(self):
self._data_dir = "${tc.dataPath}/"
def load_data(self, batch_size):
train_h5, test_h5 = self.load_h5_files()
data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
train_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
test_iter = None
if test_h5 != None:
test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
test_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
return train_iter, test_iter, data_mean, data_std
def load_h5_files(self):
train_h5 = None
test_h5 = None
train_path = self._data_dir + "train.h5"
test_path = self._data_dir + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
test_iter = None
if os.path.isfile(test_path):
test_h5 = h5py.File(test_path, 'r')
if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
else:
logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
return train_h5, test_h5
else:
logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
sys.exit(1)
\ No newline at end of file
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
0 → 100644
View file @
2dd5aae6
import mxnet as mx
import logging
import numpy as np
import time
import os
import shutil
from mxnet import gluon, autograd, nd
class CNNSupervisedTrainer(object):
def __init__(self, data_loader, net_constructor, net=None):
self._data_loader = data_loader
self._net_creator = net_constructor
self._net = net
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
optimizer='adam',
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
elif context == 'cpu':
mx_context = mx.cpu()
else:
logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
del optimizer_params['weight_decay']
if 'learning_rate_decay' in optimizer_params:
min_learning_rate = 1e-08
if 'learning_rate_minimum' in optimizer_params:
min_learning_rate = optimizer_params['learning_rate_minimum']
del optimizer_params['learning_rate_minimum']
optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
optimizer_params['step_size'],
factor=optimizer_params['learning_rate_decay'],
stop_factor_lr=min_learning_rate)
del optimizer_params['step_size']
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
if self._net is None:
if normalize:
self._net_creator.construct(
context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
else:
self._net_creator.construct(context=mx_context)
begin_epoch = 0
if load_checkpoint:
begin_epoch = self._net_creator.load(mx_context)
else:
if os.path.isdir(self._net_creator._model_dir_):
shutil.rmtree(self._net_creator._model_dir_)
self._net = self._net_creator.net
try:
os.makedirs(self._net_creator._model_dir_)
except OSError:
if not os.path.isdir(self._net_creator._model_dir_):
raise
trainer = mx.gluon.Trainer(self._net.collect_params(), optimizer, optimizer_params)
if self._net.last_layer == 'softmax':
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
elif self._net.last_layer == 'sigmoid':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif self._net.last_layer == 'linear':
loss_function = mx.gluon.loss.L2Loss()
else: # TODO: Change default?
loss_function = mx.gluon.loss.L2Loss()
logging.warning("Invalid last_layer, defaulting to L2 loss")
speed_period = 50
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
with autograd.record():
output = self._net(data)
loss = loss_function(output, label)
loss.backward()
trainer.step(batch_size)
if tic is None:
tic = time.time()
else:
if batch_i % speed_period == 0:
try:
speed = speed_period * batch_size / (time.time() - tic)
except ZeroDivisionError:
speed = float("inf")
logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
tic = time.time()
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self._net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(test_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self._net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if (epoch - begin_epoch) % checkpoint_period == 0:
self._net.save_parameters(self.parameter_path() + '-' + str(epoch).zfill(4) + '.params')
self._net.save_parameters(self.parameter_path() + '-' + str(num_epoch + begin_epoch).zfill(4) + '.params')
self._net.export(self.parameter_path() + '_newest', epoch=0)
def parameter_path(self):
return self._net_creator._model_dir_ + self._net_creator._model_prefix_
\ No newline at end of file
src/main/resources/templates/gluon/CNNTrainer.ftl
View file @
2dd5aae6
import logging
import mxnet as mx
import supervised_trainer
<#list configurations as config>
import CNNCreator_${config.instanceName}
import CNNDataLoader_${config.instanceName}
</#list>
if __name__ == "__main__":
...
...
@@ -11,8 +13,12 @@ if __name__ == "__main__":
logger.addHandler(handler)
<#list configurations as config>
${config.instanceName} = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
${config.instanceName}.train(
${config.instanceName}_creator = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
${config.instanceName}_loader = CNNDataLoader_${config.instanceName}.${config.instanceName}DataLoader()
${config.instanceName}_trainer = supervised_trainer.CNNSupervisedTrainer(${config.instanceName}_loader,
${config.instanceName}_creator)
${config.instanceName}_trainer.train(
<#if (config.batchSize)??>
batch_size=${config.batchSize},
</#if>
...
...
src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java
View file @
2dd5aae6
...
...
@@ -54,6 +54,7 @@ public class GenerationTest extends AbstractSymtabTest {
Arrays
.
asList
(
"CNNCreator_CifarClassifierNetwork.py"
,
"CNNNet_CifarClassifierNetwork.py"
,
"CNNDataLoader_CifarClassifierNetwork.py"
,
"CNNPredictor_CifarClassifierNetwork.h"
,
"execute_CifarClassifierNetwork"
,
"CNNBufferFile.h"
));
...
...
@@ -72,6 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
Arrays
.
asList
(
"CNNCreator_Alexnet.py"
,
"CNNNet_Alexnet.py"
,
"CNNDataLoader_Alexnet.py"
,
"CNNPredictor_Alexnet.h"
,
"execute_Alexnet"
));
}
...
...
@@ -89,6 +91,7 @@ public class GenerationTest extends AbstractSymtabTest {
Arrays
.
asList
(
"CNNCreator_VGG16.py"
,
"CNNNet_VGG16.py"
,
"CNNDataLoader_VGG16.py"
,
"CNNPredictor_VGG16.h"
,
"execute_VGG16"
));
}
...
...
@@ -130,7 +133,8 @@ public class GenerationTest extends AbstractSymtabTest {
Paths
.
get
(
"./target/generated-sources-cnnarch"
),
Paths
.
get
(
"./src/test/resources/target_code"
),
Arrays
.
asList
(
"CNNTrainer_fullConfig.py"
));
"CNNTrainer_fullConfig.py"
,
"supervised_trainer.py"
));
}
@Test
...
...
@@ -146,7 +150,8 @@ public class GenerationTest extends AbstractSymtabTest {
Paths
.
get
(
"./target/generated-sources-cnnarch"
),
Paths
.
get
(
"./src/test/resources/target_code"
),
Arrays
.
asList
(
"CNNTrainer_simpleConfig.py"
));
"CNNTrainer_simpleConfig.py"
,
"supervised_trainer.py"
));
}
@Test
...
...
@@ -161,7 +166,8 @@ public class GenerationTest extends AbstractSymtabTest {
Paths
.
get
(
"./target/generated-sources-cnnarch"
),
Paths
.
get
(
"./src/test/resources/target_code"
),
Arrays
.
asList
(
"CNNTrainer_emptyConfig.py"
));
"CNNTrainer_emptyConfig.py"
,
"supervised_trainer.py"
));
}
...
...
src/test/resources/target_code/CNNCreator_Alexnet.py
View file @
2dd5aae6
import
mxnet
as
mx
import
logging
import
os
import
errno
import
shutil
import
h5py
import
sys
import
numpy
as
np
import
time
from
mxnet
import
gluon
,
autograd
,
nd
from
CNNNet_Alexnet
import
Net
@
mx
.
init
.
register
class
MyConstant
(
mx
.
init
.
Initializer
):
def
__init__
(
self
,
value
):
super
(
MyConstant
,
self
).
__init__
(
value
=
value
)
self
.
value
=
value
def
_init_weight
(
self
,
_
,
arr
):
arr
[:]
=
mx
.
nd
.
array
(
self
.
value
)
class
CNNCreator_Alexnet
:
_data_dir_
=
"data/Alexnet/"
_model_dir_
=
"model/Alexnet/"
_model_prefix_
=
"model"
_input_names_
=
[
'data'
]
_input_shapes_
=
[(
3
,
224
,
224
)]
_output_names_
=
[
'predictions_label'
]
def
__init__
(
self
):
self
.
weight_initializer
=
mx
.
init
.
Normal
()
...
...
@@ -60,176 +41,6 @@ class CNNCreator_Alexnet:
return
lastEpoch
def
load_data
(
self
,
batch_size
):
train_h5
,
test_h5
=
self
.
load_h5_files
()
data_mean
=
train_h5
[
self
.
_input_names_
[
0
]][:].
mean
(
axis
=
0
)
data_std
=
train_h5
[
self
.
_input_names_
[
0
]][:].
std
(
axis
=
0
)
+
1e-5
train_iter
=
mx
.
io
.
NDArrayIter
(
train_h5
[
self
.
_input_names_
[
0
]],
train_h5
[
self
.
_output_names_
[
0
]],
batch_size
=
batch_size
,
data_name
=
self
.
_input_names_
[
0
],
label_name
=
self
.
_output_names_
[
0
])
test_iter
=
None
if
test_h5
!=
None
:
test_iter
=
mx
.
io
.
NDArrayIter
(
test_h5
[
self
.
_input_names_
[
0
]],
test_h5
[
self
.
_output_names_
[
0
]],
batch_size
=
batch_size
,
data_name
=
self
.
_input_names_
[
0
],
label_name
=
self
.
_output_names_
[
0
])
return
train_iter
,
test_iter
,
data_mean
,
data_std
def
load_h5_files
(
self
):
train_h5
=
None
test_h5
=
None
train_path
=
self
.
_data_dir_
+
"train.h5"
test_path
=
self
.
_data_dir_
+
"test.h5"
if
os
.
path
.
isfile
(
train_path
):
train_h5
=
h5py
.
File
(
train_path
,
'r'
)
if
not
(
self
.
_input_names_
[
0
]
in
train_h5
and
self
.
_output_names_
[
0
]
in
train_h5
):
logging
.
error
(
"The HDF5 file '"
+
os
.
path
.
abspath
(
train_path
)
+
"' has to contain the datasets: "
+
"'"
+
self
.
_input_names_
[
0
]
+
"', '"
+
self
.
_output_names_
[
0
]
+
"'"
)
sys
.
exit
(
1
)
test_iter
=
None
if
os
.
path
.
isfile
(
test_path
):
test_h5
=
h5py
.
File
(
test_path
,
'r'
)
if
not
(
self
.
_input_names_
[
0
]
in
test_h5
and
self
.
_output_names_
[
0
]
in
test_h5
):
logging
.
error
(
"The HDF5 file '"
+
os
.
path
.
abspath
(
test_path
)
+
"' has to contain the datasets: "
+
"'"
+
self
.
_input_names_
[
0
]
+
"', '"
+
self
.
_output_names_
[
0
]
+
"'"
)
sys
.
exit
(
1
)
else
:
logging
.
warning
(
"Couldn't load test set. File '"
+
os
.
path
.
abspath
(
test_path
)
+
"' does not exist."
)
return
train_h5
,
test_h5
else
:
logging
.
error
(
"Data loading failure. File '"
+
os
.
path
.
abspath
(
train_path
)
+
"' does not exist."
)
sys
.
exit
(
1
)
def
train
(
self
,
batch_size
=
64
,
num_epoch
=
10
,
eval_metric
=
'acc'
,
optimizer
=
'adam'
,
optimizer_params
=
((
'learning_rate'
,
0.001
),),
load_checkpoint
=
True
,
context
=
'gpu'
,
checkpoint_period
=
5
,
normalize
=
True
):
if
context
==
'gpu'
:
mx_context
=
mx
.
gpu
()
elif
context
==
'cpu'
:
mx_context
=
mx
.
cpu
()
else
:
logging
.
error
(
"Context argument is '"
+
context
+
"'. Only 'cpu' and 'gpu are valid arguments'."
)