From e5d1c8d6e82e1cbb7c4d632efc38d659471e043d Mon Sep 17 00:00:00 2001
From: Nicola Gatto <nicola.gatto@rwth-aachen.de>
Date: Mon, 8 Apr 2019 15:36:42 +0200
Subject: [PATCH] Move generated training files
---
pom.xml | 2 +-
.../cnnarch/gluongenerator/CNNArch2Gluon.java | 3 +
.../gluongenerator/CNNTrain2Gluon.java | 11 +-
.../resources/templates/gluon/CNNCreator.ftl | 189 ------------------
.../templates/gluon/CNNDataLoader.ftl | 57 ++++++
.../templates/gluon/CNNSupervisedTrainer.ftl | 141 +++++++++++++
.../resources/templates/gluon/CNNTrainer.ftl | 10 +-
.../gluongenerator/GenerationTest.java | 12 +-
.../target_code/CNNCreator_Alexnet.py | 189 ------------------
.../CNNCreator_CifarClassifierNetwork.py | 189 ------------------
.../resources/target_code/CNNCreator_VGG16.py | 189 ------------------
.../target_code/CNNDataLoader_Alexnet.py | 57 ++++++
.../CNNDataLoader_CifarClassifierNetwork.py | 57 ++++++
.../target_code/CNNDataLoader_VGG16.py | 57 ++++++
.../target_code/CNNTrainer_emptyConfig.py | 10 +-
.../target_code/CNNTrainer_fullConfig.py | 10 +-
.../target_code/CNNTrainer_simpleConfig.py | 10 +-
.../target_code/supervised_trainer.py | 141 +++++++++++++
18 files changed, 564 insertions(+), 770 deletions(-)
create mode 100644 src/main/resources/templates/gluon/CNNDataLoader.ftl
create mode 100644 src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
create mode 100644 src/test/resources/target_code/CNNDataLoader_Alexnet.py
create mode 100644 src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py
create mode 100644 src/test/resources/target_code/CNNDataLoader_VGG16.py
create mode 100644 src/test/resources/target_code/supervised_trainer.py
diff --git a/pom.xml b/pom.xml
index 6b014ce0..0cb7bc93 100644
--- a/pom.xml
+++ b/pom.xml
@@ -8,7 +8,7 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-gluon-generator</artifactId>
- <version>0.1.5</version>
+ <version>0.1.6</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
diff --git a/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java b/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java
index 652e09db..e0a185a9 100644
--- a/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java
+++ b/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java
@@ -47,6 +47,9 @@ public class CNNArch2Gluon extends CNNArch2MxNet {
temp = archTc.process("CNNNet", Target.PYTHON);
fileContentMap.put(temp.getKey(), temp.getValue());
+ temp = archTc.process("CNNDataLoader", Target.PYTHON);
+ fileContentMap.put(temp.getKey(), temp.getValue());
+
temp = archTc.process("CNNCreator", Target.PYTHON);
fileContentMap.put(temp.getKey(), temp.getValue());
diff --git a/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java b/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java
index 8be4e1f5..a59beb14 100644
--- a/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java
+++ b/src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java
@@ -21,7 +21,14 @@ public class CNNTrain2Gluon extends CNNTrain2MxNet {
configDataList.add(configData);
Map<String, Object> ftlContext = Collections.singletonMap("configurations", configDataList);
- String templateContent = templateConfiguration.processTemplate(ftlContext, "CNNTrainer.ftl");
- return Collections.singletonMap("CNNTrainer_" + getInstanceName() + ".py", templateContent);
+ Map<String, String> fileContentMap = new HashMap<>();
+
+ String cnnTrainTemplateContent = templateConfiguration.processTemplate(ftlContext, "CNNTrainer.ftl");
+ fileContentMap.put("CNNTrainer_" + getInstanceName() + ".py", cnnTrainTemplateContent);
+
+ String cnnSupervisedTrainerContent = templateConfiguration.processTemplate(ftlContext, "CNNSupervisedTrainer.ftl");
+ fileContentMap.put("supervised_trainer.py", cnnSupervisedTrainerContent);
+
+ return fileContentMap;
}
}
\ No newline at end of file
diff --git a/src/main/resources/templates/gluon/CNNCreator.ftl b/src/main/resources/templates/gluon/CNNCreator.ftl
index a9f6e3c9..fd2e4061 100644
--- a/src/main/resources/templates/gluon/CNNCreator.ftl
+++ b/src/main/resources/templates/gluon/CNNCreator.ftl
@@ -1,31 +1,12 @@
import mxnet as mx
import logging
import os
-import errno
-import shutil
-import h5py
-import sys
-import numpy as np
-import time
-from mxnet import gluon, autograd, nd
from CNNNet_${tc.fullArchitectureName} import Net
-@mx.init.register
-class MyConstant(mx.init.Initializer):
- def __init__(self, value):
- super(MyConstant, self).__init__(value=value)
- self.value = value
- def _init_weight(self, _, arr):
- arr[:] = mx.nd.array(self.value)
-
class ${tc.fileNameWithoutEnding}:
-
- _data_dir_ = "${tc.dataPath}/"
_model_dir_ = "model/${tc.componentName}/"
_model_prefix_ = "model"
- _input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
_input_shapes_ = [<#list tc.architecture.inputs as input>(${tc.join(input.definition.type.dimensions, ",")})</#list>]
- _output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
def __init__(self):
self.weight_initializer = mx.init.Normal()
@@ -60,176 +41,6 @@ class ${tc.fileNameWithoutEnding}:
return lastEpoch
- def load_data(self, batch_size):
- train_h5, test_h5 = self.load_h5_files()
-
- data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
- data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
-
- train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
- train_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- test_iter = None
- if test_h5 != None:
- test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
- test_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- return train_iter, test_iter, data_mean, data_std
-
- def load_h5_files(self):
- train_h5 = None
- test_h5 = None
- train_path = self._data_dir_ + "train.h5"
- test_path = self._data_dir_ + "test.h5"
- if os.path.isfile(train_path):
- train_h5 = h5py.File(train_path, 'r')
- if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
- logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- test_iter = None
- if os.path.isfile(test_path):
- test_h5 = h5py.File(test_path, 'r')
- if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
- logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- else:
- logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
- return train_h5, test_h5
- else:
- logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
- sys.exit(1)
-
-
- def train(self, batch_size=64,
- num_epoch=10,
- eval_metric='acc',
- optimizer='adam',
- optimizer_params=(('learning_rate', 0.001),),
- load_checkpoint=True,
- context='gpu',
- checkpoint_period=5,
- normalize=True):
- if context == 'gpu':
- mx_context = mx.gpu()
- elif context == 'cpu':
- mx_context = mx.cpu()
- else:
- logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
-
- if 'weight_decay' in optimizer_params:
- optimizer_params['wd'] = optimizer_params['weight_decay']
- del optimizer_params['weight_decay']
- if 'learning_rate_decay' in optimizer_params:
- min_learning_rate = 1e-08
- if 'learning_rate_minimum' in optimizer_params:
- min_learning_rate = optimizer_params['learning_rate_minimum']
- del optimizer_params['learning_rate_minimum']
- optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
- optimizer_params['step_size'],
- factor=optimizer_params['learning_rate_decay'],
- stop_factor_lr=min_learning_rate)
- del optimizer_params['step_size']
- del optimizer_params['learning_rate_decay']
-
-
- train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
- if self.net == None:
- if normalize:
- self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
- else:
- self.construct(context=mx_context)
-
- begin_epoch = 0
- if load_checkpoint:
- begin_epoch = self.load(mx_context)
- else:
- if os.path.isdir(self._model_dir_):
- shutil.rmtree(self._model_dir_)
-
- try:
- os.makedirs(self._model_dir_)
- except OSError:
- if not os.path.isdir(self._model_dir_):
- raise
-
- trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
-
- if self.net.last_layer == 'softmax':
- loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
- elif self.net.last_layer == 'sigmoid':
- loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
- elif self.net.last_layer == 'linear':
- loss_function = mx.gluon.loss.L2Loss()
- else: # TODO: Change default?
- loss_function = mx.gluon.loss.L2Loss()
- logging.warning("Invalid last_layer, defaulting to L2 loss")
-
- speed_period = 50
- tic = None
-
- for epoch in range(begin_epoch, begin_epoch + num_epoch):
- train_iter.reset()
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- with autograd.record():
- output = self.net(data)
- loss = loss_function(output, label)
-
- loss.backward()
- trainer.step(batch_size)
-
- if tic is None:
- tic = time.time()
- else:
- if batch_i % speed_period == 0:
- try:
- speed = speed_period * batch_size / (time.time() - tic)
- except ZeroDivisionError:
- speed = float("inf")
-
- logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
-
- tic = time.time()
-
- tic = None
-
- train_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- train_metric_score = metric.get()[1]
-
- test_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(test_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- test_metric_score = metric.get()[1]
-
- logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
-
- if (epoch - begin_epoch) % checkpoint_period == 0:
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
-
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
- + str(num_epoch + begin_epoch).zfill(4) + '.params')
- self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
-
-
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
diff --git a/src/main/resources/templates/gluon/CNNDataLoader.ftl b/src/main/resources/templates/gluon/CNNDataLoader.ftl
new file mode 100644
index 00000000..bc2a6a16
--- /dev/null
+++ b/src/main/resources/templates/gluon/CNNDataLoader.ftl
@@ -0,0 +1,57 @@
+import os
+import h5py
+import mxnet as mx
+import logging
+import sys
+
+class ${tc.fullArchitectureName}DataLoader:
+ _input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
+ _output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
+
+ def __init__(self):
+ self._data_dir = "${tc.dataPath}/"
+
+ def load_data(self, batch_size):
+ train_h5, test_h5 = self.load_h5_files()
+
+ data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
+ data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
+
+ train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
+ train_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ test_iter = None
+ if test_h5 != None:
+ test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
+ test_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ return train_iter, test_iter, data_mean, data_std
+
+ def load_h5_files(self):
+ train_h5 = None
+ test_h5 = None
+ train_path = self._data_dir + "train.h5"
+ test_path = self._data_dir + "test.h5"
+ if os.path.isfile(train_path):
+ train_h5 = h5py.File(train_path, 'r')
+ if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ test_iter = None
+ if os.path.isfile(test_path):
+ test_h5 = h5py.File(test_path, 'r')
+ if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ else:
+ logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
+ return train_h5, test_h5
+ else:
+ logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
+ sys.exit(1)
\ No newline at end of file
diff --git a/src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl b/src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
new file mode 100644
index 00000000..6a471488
--- /dev/null
+++ b/src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
@@ -0,0 +1,141 @@
+import mxnet as mx
+import logging
+import numpy as np
+import time
+import os
+import shutil
+from mxnet import gluon, autograd, nd
+
+class CNNSupervisedTrainer(object):
+ def __init__(self, data_loader, net_constructor, net=None):
+ self._data_loader = data_loader
+ self._net_creator = net_constructor
+ self._net = net
+
+ def train(self, batch_size=64,
+ num_epoch=10,
+ eval_metric='acc',
+ optimizer='adam',
+ optimizer_params=(('learning_rate', 0.001),),
+ load_checkpoint=True,
+ context='gpu',
+ checkpoint_period=5,
+ normalize=True):
+ if context == 'gpu':
+ mx_context = mx.gpu()
+ elif context == 'cpu':
+ mx_context = mx.cpu()
+ else:
+ logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
+
+ if 'weight_decay' in optimizer_params:
+ optimizer_params['wd'] = optimizer_params['weight_decay']
+ del optimizer_params['weight_decay']
+ if 'learning_rate_decay' in optimizer_params:
+ min_learning_rate = 1e-08
+ if 'learning_rate_minimum' in optimizer_params:
+ min_learning_rate = optimizer_params['learning_rate_minimum']
+ del optimizer_params['learning_rate_minimum']
+ optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
+ optimizer_params['step_size'],
+ factor=optimizer_params['learning_rate_decay'],
+ stop_factor_lr=min_learning_rate)
+ del optimizer_params['step_size']
+ del optimizer_params['learning_rate_decay']
+
+
+ train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
+ if self._net is None:
+ if normalize:
+ self._net_creator.construct(
+ context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
+ else:
+ self._net_creator.construct(context=mx_context)
+
+ begin_epoch = 0
+ if load_checkpoint:
+ begin_epoch = self._net_creator.load(mx_context)
+ else:
+ if os.path.isdir(self._net_creator._model_dir_):
+ shutil.rmtree(self._net_creator._model_dir_)
+
+ self._net = self._net_creator.net
+
+ try:
+ os.makedirs(self._net_creator._model_dir_)
+ except OSError:
+ if not os.path.isdir(self._net_creator._model_dir_):
+ raise
+
+ trainer = mx.gluon.Trainer(self._net.collect_params(), optimizer, optimizer_params)
+
+ if self._net.last_layer == 'softmax':
+ loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
+ elif self._net.last_layer == 'sigmoid':
+ loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
+ elif self._net.last_layer == 'linear':
+ loss_function = mx.gluon.loss.L2Loss()
+ else: # TODO: Change default?
+ loss_function = mx.gluon.loss.L2Loss()
+ logging.warning("Invalid last_layer, defaulting to L2 loss")
+
+ speed_period = 50
+ tic = None
+
+ for epoch in range(begin_epoch, begin_epoch + num_epoch):
+ train_iter.reset()
+ for batch_i, batch in enumerate(train_iter):
+ data = batch.data[0].as_in_context(mx_context)
+ label = batch.label[0].as_in_context(mx_context)
+ with autograd.record():
+ output = self._net(data)
+ loss = loss_function(output, label)
+
+ loss.backward()
+ trainer.step(batch_size)
+
+ if tic is None:
+ tic = time.time()
+ else:
+ if batch_i % speed_period == 0:
+ try:
+ speed = speed_period * batch_size / (time.time() - tic)
+ except ZeroDivisionError:
+ speed = float("inf")
+
+ logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
+
+ tic = time.time()
+
+ tic = None
+
+ train_iter.reset()
+ metric = mx.metric.create(eval_metric)
+ for batch_i, batch in enumerate(train_iter):
+ data = batch.data[0].as_in_context(mx_context)
+ label = batch.label[0].as_in_context(mx_context)
+ output = self._net(data)
+ predictions = mx.nd.argmax(output, axis=1)
+ metric.update(preds=predictions, labels=label)
+ train_metric_score = metric.get()[1]
+
+ test_iter.reset()
+ metric = mx.metric.create(eval_metric)
+ for batch_i, batch in enumerate(test_iter):
+ data = batch.data[0].as_in_context(mx_context)
+ label = batch.label[0].as_in_context(mx_context)
+ output = self._net(data)
+ predictions = mx.nd.argmax(output, axis=1)
+ metric.update(preds=predictions, labels=label)
+ test_metric_score = metric.get()[1]
+
+ logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
+
+ if (epoch - begin_epoch) % checkpoint_period == 0:
+ self._net.save_parameters(self.parameter_path() + '-' + str(epoch).zfill(4) + '.params')
+
+ self._net.save_parameters(self.parameter_path() + '-' + str(num_epoch + begin_epoch).zfill(4) + '.params')
+ self._net.export(self.parameter_path() + '_newest', epoch=0)
+
+ def parameter_path(self):
+ return self._net_creator._model_dir_ + self._net_creator._model_prefix_
\ No newline at end of file
diff --git a/src/main/resources/templates/gluon/CNNTrainer.ftl b/src/main/resources/templates/gluon/CNNTrainer.ftl
index de2e1fec..cca29dbe 100644
--- a/src/main/resources/templates/gluon/CNNTrainer.ftl
+++ b/src/main/resources/templates/gluon/CNNTrainer.ftl
@@ -1,7 +1,9 @@
import logging
import mxnet as mx
+import supervised_trainer
<#list configurations as config>
import CNNCreator_${config.instanceName}
+import CNNDataLoader_${config.instanceName}
</#list>
if __name__ == "__main__":
@@ -11,8 +13,12 @@ if __name__ == "__main__":
logger.addHandler(handler)
<#list configurations as config>
- ${config.instanceName} = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
- ${config.instanceName}.train(
+ ${config.instanceName}_creator = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
+ ${config.instanceName}_loader = CNNDataLoader_${config.instanceName}.${config.instanceName}DataLoader()
+ ${config.instanceName}_trainer = supervised_trainer.CNNSupervisedTrainer(${config.instanceName}_loader,
+ ${config.instanceName}_creator)
+
+ ${config.instanceName}_trainer.train(
<#if (config.batchSize)??>
batch_size=${config.batchSize},
</#if>
diff --git a/src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java b/src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java
index 30e242da..545ad921 100644
--- a/src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java
+++ b/src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java
@@ -54,6 +54,7 @@ public class GenerationTest extends AbstractSymtabTest {
Arrays.asList(
"CNNCreator_CifarClassifierNetwork.py",
"CNNNet_CifarClassifierNetwork.py",
+ "CNNDataLoader_CifarClassifierNetwork.py",
"CNNPredictor_CifarClassifierNetwork.h",
"execute_CifarClassifierNetwork",
"CNNBufferFile.h"));
@@ -72,6 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
Arrays.asList(
"CNNCreator_Alexnet.py",
"CNNNet_Alexnet.py",
+ "CNNDataLoader_Alexnet.py",
"CNNPredictor_Alexnet.h",
"execute_Alexnet"));
}
@@ -89,6 +91,7 @@ public class GenerationTest extends AbstractSymtabTest {
Arrays.asList(
"CNNCreator_VGG16.py",
"CNNNet_VGG16.py",
+ "CNNDataLoader_VGG16.py",
"CNNPredictor_VGG16.h",
"execute_VGG16"));
}
@@ -130,7 +133,8 @@ public class GenerationTest extends AbstractSymtabTest {
Paths.get("./target/generated-sources-cnnarch"),
Paths.get("./src/test/resources/target_code"),
Arrays.asList(
- "CNNTrainer_fullConfig.py"));
+ "CNNTrainer_fullConfig.py",
+ "supervised_trainer.py"));
}
@Test
@@ -146,7 +150,8 @@ public class GenerationTest extends AbstractSymtabTest {
Paths.get("./target/generated-sources-cnnarch"),
Paths.get("./src/test/resources/target_code"),
Arrays.asList(
- "CNNTrainer_simpleConfig.py"));
+ "CNNTrainer_simpleConfig.py",
+ "supervised_trainer.py"));
}
@Test
@@ -161,7 +166,8 @@ public class GenerationTest extends AbstractSymtabTest {
Paths.get("./target/generated-sources-cnnarch"),
Paths.get("./src/test/resources/target_code"),
Arrays.asList(
- "CNNTrainer_emptyConfig.py"));
+ "CNNTrainer_emptyConfig.py",
+ "supervised_trainer.py"));
}
diff --git a/src/test/resources/target_code/CNNCreator_Alexnet.py b/src/test/resources/target_code/CNNCreator_Alexnet.py
index 361567e2..1b2befae 100644
--- a/src/test/resources/target_code/CNNCreator_Alexnet.py
+++ b/src/test/resources/target_code/CNNCreator_Alexnet.py
@@ -1,31 +1,12 @@
import mxnet as mx
import logging
import os
-import errno
-import shutil
-import h5py
-import sys
-import numpy as np
-import time
-from mxnet import gluon, autograd, nd
from CNNNet_Alexnet import Net
-@mx.init.register
-class MyConstant(mx.init.Initializer):
- def __init__(self, value):
- super(MyConstant, self).__init__(value=value)
- self.value = value
- def _init_weight(self, _, arr):
- arr[:] = mx.nd.array(self.value)
-
class CNNCreator_Alexnet:
-
- _data_dir_ = "data/Alexnet/"
_model_dir_ = "model/Alexnet/"
_model_prefix_ = "model"
- _input_names_ = ['data']
_input_shapes_ = [(3,224,224)]
- _output_names_ = ['predictions_label']
def __init__(self):
self.weight_initializer = mx.init.Normal()
@@ -60,176 +41,6 @@ class CNNCreator_Alexnet:
return lastEpoch
- def load_data(self, batch_size):
- train_h5, test_h5 = self.load_h5_files()
-
- data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
- data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
-
- train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
- train_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- test_iter = None
- if test_h5 != None:
- test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
- test_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- return train_iter, test_iter, data_mean, data_std
-
- def load_h5_files(self):
- train_h5 = None
- test_h5 = None
- train_path = self._data_dir_ + "train.h5"
- test_path = self._data_dir_ + "test.h5"
- if os.path.isfile(train_path):
- train_h5 = h5py.File(train_path, 'r')
- if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
- logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- test_iter = None
- if os.path.isfile(test_path):
- test_h5 = h5py.File(test_path, 'r')
- if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
- logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- else:
- logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
- return train_h5, test_h5
- else:
- logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
- sys.exit(1)
-
-
- def train(self, batch_size=64,
- num_epoch=10,
- eval_metric='acc',
- optimizer='adam',
- optimizer_params=(('learning_rate', 0.001),),
- load_checkpoint=True,
- context='gpu',
- checkpoint_period=5,
- normalize=True):
- if context == 'gpu':
- mx_context = mx.gpu()
- elif context == 'cpu':
- mx_context = mx.cpu()
- else:
- logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
-
- if 'weight_decay' in optimizer_params:
- optimizer_params['wd'] = optimizer_params['weight_decay']
- del optimizer_params['weight_decay']
- if 'learning_rate_decay' in optimizer_params:
- min_learning_rate = 1e-08
- if 'learning_rate_minimum' in optimizer_params:
- min_learning_rate = optimizer_params['learning_rate_minimum']
- del optimizer_params['learning_rate_minimum']
- optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
- optimizer_params['step_size'],
- factor=optimizer_params['learning_rate_decay'],
- stop_factor_lr=min_learning_rate)
- del optimizer_params['step_size']
- del optimizer_params['learning_rate_decay']
-
-
- train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
- if self.net == None:
- if normalize:
- self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
- else:
- self.construct(context=mx_context)
-
- begin_epoch = 0
- if load_checkpoint:
- begin_epoch = self.load(mx_context)
- else:
- if os.path.isdir(self._model_dir_):
- shutil.rmtree(self._model_dir_)
-
- try:
- os.makedirs(self._model_dir_)
- except OSError:
- if not os.path.isdir(self._model_dir_):
- raise
-
- trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
-
- if self.net.last_layer == 'softmax':
- loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
- elif self.net.last_layer == 'sigmoid':
- loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
- elif self.net.last_layer == 'linear':
- loss_function = mx.gluon.loss.L2Loss()
- else: # TODO: Change default?
- loss_function = mx.gluon.loss.L2Loss()
- logging.warning("Invalid last_layer, defaulting to L2 loss")
-
- speed_period = 50
- tic = None
-
- for epoch in range(begin_epoch, begin_epoch + num_epoch):
- train_iter.reset()
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- with autograd.record():
- output = self.net(data)
- loss = loss_function(output, label)
-
- loss.backward()
- trainer.step(batch_size)
-
- if tic is None:
- tic = time.time()
- else:
- if batch_i % speed_period == 0:
- try:
- speed = speed_period * batch_size / (time.time() - tic)
- except ZeroDivisionError:
- speed = float("inf")
-
- logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
-
- tic = time.time()
-
- tic = None
-
- train_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- train_metric_score = metric.get()[1]
-
- test_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(test_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- test_metric_score = metric.get()[1]
-
- logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
-
- if (epoch - begin_epoch) % checkpoint_period == 0:
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
-
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
- + str(num_epoch + begin_epoch).zfill(4) + '.params')
- self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
-
-
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
diff --git a/src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py b/src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
index 65b8a87c..5d63b10b 100644
--- a/src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
+++ b/src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
@@ -1,31 +1,12 @@
import mxnet as mx
import logging
import os
-import errno
-import shutil
-import h5py
-import sys
-import numpy as np
-import time
-from mxnet import gluon, autograd, nd
from CNNNet_CifarClassifierNetwork import Net
-@mx.init.register
-class MyConstant(mx.init.Initializer):
- def __init__(self, value):
- super(MyConstant, self).__init__(value=value)
- self.value = value
- def _init_weight(self, _, arr):
- arr[:] = mx.nd.array(self.value)
-
class CNNCreator_CifarClassifierNetwork:
-
- _data_dir_ = "data/CifarClassifierNetwork/"
_model_dir_ = "model/CifarClassifierNetwork/"
_model_prefix_ = "model"
- _input_names_ = ['data']
_input_shapes_ = [(3,32,32)]
- _output_names_ = ['softmax_label']
def __init__(self):
self.weight_initializer = mx.init.Normal()
@@ -60,176 +41,6 @@ class CNNCreator_CifarClassifierNetwork:
return lastEpoch
- def load_data(self, batch_size):
- train_h5, test_h5 = self.load_h5_files()
-
- data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
- data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
-
- train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
- train_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- test_iter = None
- if test_h5 != None:
- test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
- test_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- return train_iter, test_iter, data_mean, data_std
-
- def load_h5_files(self):
- train_h5 = None
- test_h5 = None
- train_path = self._data_dir_ + "train.h5"
- test_path = self._data_dir_ + "test.h5"
- if os.path.isfile(train_path):
- train_h5 = h5py.File(train_path, 'r')
- if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
- logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- test_iter = None
- if os.path.isfile(test_path):
- test_h5 = h5py.File(test_path, 'r')
- if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
- logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- else:
- logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
- return train_h5, test_h5
- else:
- logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
- sys.exit(1)
-
-
- def train(self, batch_size=64,
- num_epoch=10,
- eval_metric='acc',
- optimizer='adam',
- optimizer_params=(('learning_rate', 0.001),),
- load_checkpoint=True,
- context='gpu',
- checkpoint_period=5,
- normalize=True):
- if context == 'gpu':
- mx_context = mx.gpu()
- elif context == 'cpu':
- mx_context = mx.cpu()
- else:
- logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
-
- if 'weight_decay' in optimizer_params:
- optimizer_params['wd'] = optimizer_params['weight_decay']
- del optimizer_params['weight_decay']
- if 'learning_rate_decay' in optimizer_params:
- min_learning_rate = 1e-08
- if 'learning_rate_minimum' in optimizer_params:
- min_learning_rate = optimizer_params['learning_rate_minimum']
- del optimizer_params['learning_rate_minimum']
- optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
- optimizer_params['step_size'],
- factor=optimizer_params['learning_rate_decay'],
- stop_factor_lr=min_learning_rate)
- del optimizer_params['step_size']
- del optimizer_params['learning_rate_decay']
-
-
- train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
- if self.net == None:
- if normalize:
- self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
- else:
- self.construct(context=mx_context)
-
- begin_epoch = 0
- if load_checkpoint:
- begin_epoch = self.load(mx_context)
- else:
- if os.path.isdir(self._model_dir_):
- shutil.rmtree(self._model_dir_)
-
- try:
- os.makedirs(self._model_dir_)
- except OSError:
- if not os.path.isdir(self._model_dir_):
- raise
-
- trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
-
- if self.net.last_layer == 'softmax':
- loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
- elif self.net.last_layer == 'sigmoid':
- loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
- elif self.net.last_layer == 'linear':
- loss_function = mx.gluon.loss.L2Loss()
- else: # TODO: Change default?
- loss_function = mx.gluon.loss.L2Loss()
- logging.warning("Invalid last_layer, defaulting to L2 loss")
-
- speed_period = 50
- tic = None
-
- for epoch in range(begin_epoch, begin_epoch + num_epoch):
- train_iter.reset()
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- with autograd.record():
- output = self.net(data)
- loss = loss_function(output, label)
-
- loss.backward()
- trainer.step(batch_size)
-
- if tic is None:
- tic = time.time()
- else:
- if batch_i % speed_period == 0:
- try:
- speed = speed_period * batch_size / (time.time() - tic)
- except ZeroDivisionError:
- speed = float("inf")
-
- logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
-
- tic = time.time()
-
- tic = None
-
- train_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- train_metric_score = metric.get()[1]
-
- test_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(test_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- test_metric_score = metric.get()[1]
-
- logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
-
- if (epoch - begin_epoch) % checkpoint_period == 0:
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
-
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
- + str(num_epoch + begin_epoch).zfill(4) + '.params')
- self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
-
-
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
diff --git a/src/test/resources/target_code/CNNCreator_VGG16.py b/src/test/resources/target_code/CNNCreator_VGG16.py
index b30f2dcf..af1eed2b 100644
--- a/src/test/resources/target_code/CNNCreator_VGG16.py
+++ b/src/test/resources/target_code/CNNCreator_VGG16.py
@@ -1,31 +1,12 @@
import mxnet as mx
import logging
import os
-import errno
-import shutil
-import h5py
-import sys
-import numpy as np
-import time
-from mxnet import gluon, autograd, nd
from CNNNet_VGG16 import Net
-@mx.init.register
-class MyConstant(mx.init.Initializer):
- def __init__(self, value):
- super(MyConstant, self).__init__(value=value)
- self.value = value
- def _init_weight(self, _, arr):
- arr[:] = mx.nd.array(self.value)
-
class CNNCreator_VGG16:
-
- _data_dir_ = "data/VGG16/"
_model_dir_ = "model/VGG16/"
_model_prefix_ = "model"
- _input_names_ = ['data']
_input_shapes_ = [(3,224,224)]
- _output_names_ = ['predictions_label']
def __init__(self):
self.weight_initializer = mx.init.Normal()
@@ -60,176 +41,6 @@ class CNNCreator_VGG16:
return lastEpoch
- def load_data(self, batch_size):
- train_h5, test_h5 = self.load_h5_files()
-
- data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
- data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
-
- train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
- train_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- test_iter = None
- if test_h5 != None:
- test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
- test_h5[self._output_names_[0]],
- batch_size=batch_size,
- data_name=self._input_names_[0],
- label_name=self._output_names_[0])
- return train_iter, test_iter, data_mean, data_std
-
- def load_h5_files(self):
- train_h5 = None
- test_h5 = None
- train_path = self._data_dir_ + "train.h5"
- test_path = self._data_dir_ + "test.h5"
- if os.path.isfile(train_path):
- train_h5 = h5py.File(train_path, 'r')
- if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
- logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- test_iter = None
- if os.path.isfile(test_path):
- test_h5 = h5py.File(test_path, 'r')
- if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
- logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
- + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
- sys.exit(1)
- else:
- logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
- return train_h5, test_h5
- else:
- logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
- sys.exit(1)
-
-
- def train(self, batch_size=64,
- num_epoch=10,
- eval_metric='acc',
- optimizer='adam',
- optimizer_params=(('learning_rate', 0.001),),
- load_checkpoint=True,
- context='gpu',
- checkpoint_period=5,
- normalize=True):
- if context == 'gpu':
- mx_context = mx.gpu()
- elif context == 'cpu':
- mx_context = mx.cpu()
- else:
- logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
-
- if 'weight_decay' in optimizer_params:
- optimizer_params['wd'] = optimizer_params['weight_decay']
- del optimizer_params['weight_decay']
- if 'learning_rate_decay' in optimizer_params:
- min_learning_rate = 1e-08
- if 'learning_rate_minimum' in optimizer_params:
- min_learning_rate = optimizer_params['learning_rate_minimum']
- del optimizer_params['learning_rate_minimum']
- optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
- optimizer_params['step_size'],
- factor=optimizer_params['learning_rate_decay'],
- stop_factor_lr=min_learning_rate)
- del optimizer_params['step_size']
- del optimizer_params['learning_rate_decay']
-
-
- train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
- if self.net == None:
- if normalize:
- self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
- else:
- self.construct(context=mx_context)
-
- begin_epoch = 0
- if load_checkpoint:
- begin_epoch = self.load(mx_context)
- else:
- if os.path.isdir(self._model_dir_):
- shutil.rmtree(self._model_dir_)
-
- try:
- os.makedirs(self._model_dir_)
- except OSError:
- if not os.path.isdir(self._model_dir_):
- raise
-
- trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
-
- if self.net.last_layer == 'softmax':
- loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
- elif self.net.last_layer == 'sigmoid':
- loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
- elif self.net.last_layer == 'linear':
- loss_function = mx.gluon.loss.L2Loss()
- else: # TODO: Change default?
- loss_function = mx.gluon.loss.L2Loss()
- logging.warning("Invalid last_layer, defaulting to L2 loss")
-
- speed_period = 50
- tic = None
-
- for epoch in range(begin_epoch, begin_epoch + num_epoch):
- train_iter.reset()
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- with autograd.record():
- output = self.net(data)
- loss = loss_function(output, label)
-
- loss.backward()
- trainer.step(batch_size)
-
- if tic is None:
- tic = time.time()
- else:
- if batch_i % speed_period == 0:
- try:
- speed = speed_period * batch_size / (time.time() - tic)
- except ZeroDivisionError:
- speed = float("inf")
-
- logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
-
- tic = time.time()
-
- tic = None
-
- train_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(train_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- train_metric_score = metric.get()[1]
-
- test_iter.reset()
- metric = mx.metric.create(eval_metric)
- for batch_i, batch in enumerate(test_iter):
- data = batch.data[0].as_in_context(mx_context)
- label = batch.label[0].as_in_context(mx_context)
- output = self.net(data)
- predictions = mx.nd.argmax(output, axis=1)
- metric.update(preds=predictions, labels=label)
- test_metric_score = metric.get()[1]
-
- logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
-
- if (epoch - begin_epoch) % checkpoint_period == 0:
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
-
- self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
- + str(num_epoch + begin_epoch).zfill(4) + '.params')
- self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
-
-
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
diff --git a/src/test/resources/target_code/CNNDataLoader_Alexnet.py b/src/test/resources/target_code/CNNDataLoader_Alexnet.py
new file mode 100644
index 00000000..264e219f
--- /dev/null
+++ b/src/test/resources/target_code/CNNDataLoader_Alexnet.py
@@ -0,0 +1,57 @@
+import os
+import h5py
+import mxnet as mx
+import logging
+import sys
+
+class AlexnetDataLoader:
+ _input_names_ = ['data']
+ _output_names_ = ['predictions_label']
+
+ def __init__(self):
+ self._data_dir = "data/Alexnet/"
+
+ def load_data(self, batch_size):
+ train_h5, test_h5 = self.load_h5_files()
+
+ data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
+ data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
+
+ train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
+ train_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ test_iter = None
+ if test_h5 != None:
+ test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
+ test_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ return train_iter, test_iter, data_mean, data_std
+
+ def load_h5_files(self):
+ train_h5 = None
+ test_h5 = None
+ train_path = self._data_dir + "train.h5"
+ test_path = self._data_dir + "test.h5"
+ if os.path.isfile(train_path):
+ train_h5 = h5py.File(train_path, 'r')
+ if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ test_iter = None
+ if os.path.isfile(test_path):
+ test_h5 = h5py.File(test_path, 'r')
+ if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ else:
+ logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
+ return train_h5, test_h5
+ else:
+ logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
+ sys.exit(1)
\ No newline at end of file
diff --git a/src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py b/src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py
new file mode 100644
index 00000000..3f6159a1
--- /dev/null
+++ b/src/test/resources/target_code/CNNDataLoader_CifarClassifierNetwork.py
@@ -0,0 +1,57 @@
+import os
+import h5py
+import mxnet as mx
+import logging
+import sys
+
+class CifarClassifierNetworkDataLoader:
+ _input_names_ = ['data']
+ _output_names_ = ['softmax_label']
+
+ def __init__(self):
+ self._data_dir = "data/CifarClassifierNetwork/"
+
+ def load_data(self, batch_size):
+ train_h5, test_h5 = self.load_h5_files()
+
+ data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
+ data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
+
+ train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
+ train_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ test_iter = None
+ if test_h5 != None:
+ test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
+ test_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ return train_iter, test_iter, data_mean, data_std
+
+ def load_h5_files(self):
+ train_h5 = None
+ test_h5 = None
+ train_path = self._data_dir + "train.h5"
+ test_path = self._data_dir + "test.h5"
+ if os.path.isfile(train_path):
+ train_h5 = h5py.File(train_path, 'r')
+ if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ test_iter = None
+ if os.path.isfile(test_path):
+ test_h5 = h5py.File(test_path, 'r')
+ if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ else:
+ logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
+ return train_h5, test_h5
+ else:
+ logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
+ sys.exit(1)
\ No newline at end of file
diff --git a/src/test/resources/target_code/CNNDataLoader_VGG16.py b/src/test/resources/target_code/CNNDataLoader_VGG16.py
new file mode 100644
index 00000000..6a46dba1
--- /dev/null
+++ b/src/test/resources/target_code/CNNDataLoader_VGG16.py
@@ -0,0 +1,57 @@
+import os
+import h5py
+import mxnet as mx
+import logging
+import sys
+
+class VGG16DataLoader:
+ _input_names_ = ['data']
+ _output_names_ = ['predictions_label']
+
+ def __init__(self):
+ self._data_dir = "data/VGG16/"
+
+ def load_data(self, batch_size):
+ train_h5, test_h5 = self.load_h5_files()
+
+ data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
+ data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
+
+ train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
+ train_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ test_iter = None
+ if test_h5 != None:
+ test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
+ test_h5[self._output_names_[0]],
+ batch_size=batch_size,
+ data_name=self._input_names_[0],
+ label_name=self._output_names_[0])
+ return train_iter, test_iter, data_mean, data_std
+
+ def load_h5_files(self):
+ train_h5 = None
+ test_h5 = None
+ train_path = self._data_dir + "train.h5"
+ test_path = self._data_dir + "test.h5"
+ if os.path.isfile(train_path):
+ train_h5 = h5py.File(train_path, 'r')
+ if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ test_iter = None
+ if os.path.isfile(test_path):
+ test_h5 = h5py.File(test_path, 'r')
+ if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
+ logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+ sys.exit(1)
+ else:
+ logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
+ return train_h5, test_h5
+ else:
+ logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
+ sys.exit(1)
\ No newline at end of file
diff --git a/src/test/resources/target_code/CNNTrainer_emptyConfig.py b/src/test/resources/target_code/CNNTrainer_emptyConfig.py
index 60ff25d0..acab71a4 100644
--- a/src/test/resources/target_code/CNNTrainer_emptyConfig.py
+++ b/src/test/resources/target_code/CNNTrainer_emptyConfig.py
@@ -1,6 +1,8 @@
import logging
import mxnet as mx
+import supervised_trainer
import CNNCreator_emptyConfig
+import CNNDataLoader_emptyConfig
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
@@ -8,6 +10,10 @@ if __name__ == "__main__":
handler = logging.FileHandler("train.log", "w", encoding=None, delay="true")
logger.addHandler(handler)
- emptyConfig = CNNCreator_emptyConfig.CNNCreator_emptyConfig()
- emptyConfig.train(
+ emptyConfig_creator = CNNCreator_emptyConfig.CNNCreator_emptyConfig()
+ emptyConfig_loader = CNNDataLoader_emptyConfig.emptyConfigDataLoader()
+ emptyConfig_trainer = supervised_trainer.CNNSupervisedTrainer(emptyConfig_loader,
+ emptyConfig_creator)
+
+ emptyConfig_trainer.train(
)
diff --git a/src/test/resources/target_code/CNNTrainer_fullConfig.py b/src/test/resources/target_code/CNNTrainer_fullConfig.py
index 2822ae2d..b73c5397 100644
--- a/src/test/resources/target_code/CNNTrainer_fullConfig.py
+++ b/src/test/resources/target_code/CNNTrainer_fullConfig.py
@@ -1,6 +1,8 @@
import logging
import mxnet as mx
+import supervised_trainer
import CNNCreator_fullConfig
+import CNNDataLoader_fullConfig
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
@@ -8,8 +10,12 @@ if __name__ == "__main__":
handler = logging.FileHandler("train.log", "w", encoding=None, delay="true")
logger.addHandler(handler)
- fullConfig = CNNCreator_fullConfig.CNNCreator_fullConfig()
- fullConfig.train(
+ fullConfig_creator = CNNCreator_fullConfig.CNNCreator_fullConfig()
+ fullConfig_loader = CNNDataLoader_fullConfig.fullConfigDataLoader()
+ fullConfig_trainer = supervised_trainer.CNNSupervisedTrainer(fullConfig_loader,
+ fullConfig_creator)
+
+ fullConfig_trainer.train(
batch_size=100,
num_epoch=5,
load_checkpoint=True,
diff --git a/src/test/resources/target_code/CNNTrainer_simpleConfig.py b/src/test/resources/target_code/CNNTrainer_simpleConfig.py
index bf9f0029..a3b84d28 100644
--- a/src/test/resources/target_code/CNNTrainer_simpleConfig.py
+++ b/src/test/resources/target_code/CNNTrainer_simpleConfig.py
@@ -1,6 +1,8 @@
import logging
import mxnet as mx
+import supervised_trainer
import CNNCreator_simpleConfig
+import CNNDataLoader_simpleConfig
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
@@ -8,8 +10,12 @@ if __name__ == "__main__":
handler = logging.FileHandler("train.log", "w", encoding=None, delay="true")
logger.addHandler(handler)
- simpleConfig = CNNCreator_simpleConfig.CNNCreator_simpleConfig()
- simpleConfig.train(
+ simpleConfig_creator = CNNCreator_simpleConfig.CNNCreator_simpleConfig()
+ simpleConfig_loader = CNNDataLoader_simpleConfig.simpleConfigDataLoader()
+ simpleConfig_trainer = supervised_trainer.CNNSupervisedTrainer(simpleConfig_loader,
+ simpleConfig_creator)
+
+ simpleConfig_trainer.train(
batch_size=100,
num_epoch=50,
optimizer='adam',
diff --git a/src/test/resources/target_code/supervised_trainer.py b/src/test/resources/target_code/supervised_trainer.py
new file mode 100644
index 00000000..6a471488
--- /dev/null
+++ b/src/test/resources/target_code/supervised_trainer.py
@@ -0,0 +1,141 @@
+import mxnet as mx
+import logging
+import numpy as np
+import time
+import os
+import shutil
+from mxnet import gluon, autograd, nd
+
+class CNNSupervisedTrainer(object):
+ def __init__(self, data_loader, net_constructor, net=None):
+ self._data_loader = data_loader
+ self._net_creator = net_constructor
+ self._net = net
+
+ def train(self, batch_size=64,
+ num_epoch=10,
+ eval_metric='acc',
+ optimizer='adam',
+ optimizer_params=(('learning_rate', 0.001),),
+ load_checkpoint=True,
+ context='gpu',
+ checkpoint_period=5,
+ normalize=True):
+ if context == 'gpu':
+ mx_context = mx.gpu()
+ elif context == 'cpu':
+ mx_context = mx.cpu()
+ else:
+ logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
+
+ if 'weight_decay' in optimizer_params:
+ optimizer_params['wd'] = optimizer_params['weight_decay']
+ del optimizer_params['weight_decay']
+ if 'learning_rate_decay' in optimizer_params:
+ min_learning_rate = 1e-08
+ if 'learning_rate_minimum' in optimizer_params:
+ min_learning_rate = optimizer_params['learning_rate_minimum']
+ del optimizer_params['learning_rate_minimum']
+ optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
+ optimizer_params['step_size'],
+ factor=optimizer_params['learning_rate_decay'],
+ stop_factor_lr=min_learning_rate)
+ del optimizer_params['step_size']
+ del optimizer_params['learning_rate_decay']
+
+
+ train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
+ if self._net is None:
+ if normalize:
+ self._net_creator.construct(
+ context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
+ else:
+ self._net_creator.construct(context=mx_context)
+
+ begin_epoch = 0
+ if load_checkpoint:
+ begin_epoch = self._net_creator.load(mx_context)
+ else:
+ if os.path.isdir(self._net_creator._model_dir_):
+ shutil.rmtree(self._net_creator._model_dir_)
+
+ self._net = self._net_creator.net
+
+ try:
+ os.makedirs(self._net_creator._model_dir_)
+ except OSError:
+ if not os.path.isdir(self._net_creator._model_dir_):
+ raise
+
+ trainer = mx.gluon.Trainer(self._net.collect_params(), optimizer, optimizer_params)
+
+ if self._net.last_layer == 'softmax':
+ loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
+ elif self._net.last_layer == 'sigmoid':
+ loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
+ elif self._net.last_layer == 'linear':
+ loss_function = mx.gluon.loss.L2Loss()
+ else: # TODO: Change default?
+ loss_function = mx.gluon.loss.L2Loss()
+ logging.warning("Invalid last_layer, defaulting to L2 loss")
+
+ speed_period = 50
+ tic = None
+
+ for epoch in range(begin_epoch, begin_epoch + num_epoch):
+ train_iter.reset()
+ for batch_i, batch in enumerate(train_iter):
+ data = batch.data[0].as_in_context(mx_context)
+ label = batch.label[0].as_in_context(mx_context)
+ with autograd.record():
+ output = self._net(data)
+ loss = loss_function(output, label)
+
+ loss.backward()
+ trainer.step(batch_size)
+
+ if tic is None:
+ tic = time.time()
+ else:
+ if batch_i % speed_period == 0:
+ try:
+ speed = speed_period * batch_size / (time.time() - tic)
+ except ZeroDivisionError:
+ speed = float("inf")
+
+ logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
+
+ tic = time.time()
+
+ tic = None
+
+ train_iter.reset()
+ metric = mx.metric.create(eval_metric)
+ for batch_i, batch in enumerate(train_iter):
+ data = batch.data[0].as_in_context(mx_context)
+ label = batch.label[0].as_in_context(mx_context)
+ output = self._net(data)
+ predictions = mx.nd.argmax(output, axis=1)
+ metric.update(preds=predictions, labels=label)
+ train_metric_score = metric.get()[1]
+
+ test_iter.reset()
+ metric = mx.metric.create(eval_metric)
+ for batch_i, batch in enumerate(test_iter):
+ data = batch.data[0].as_in_context(mx_context)
+ label = batch.label[0].as_in_context(mx_context)
+ output = self._net(data)
+ predictions = mx.nd.argmax(output, axis=1)
+ metric.update(preds=predictions, labels=label)
+ test_metric_score = metric.get()[1]
+
+ logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
+
+ if (epoch - begin_epoch) % checkpoint_period == 0:
+ self._net.save_parameters(self.parameter_path() + '-' + str(epoch).zfill(4) + '.params')
+
+ self._net.save_parameters(self.parameter_path() + '-' + str(num_epoch + begin_epoch).zfill(4) + '.params')
+ self._net.export(self.parameter_path() + '_newest', epoch=0)
+
+ def parameter_path(self):
+ return self._net_creator._model_dir_ + self._net_creator._model_prefix_
\ No newline at end of file
--
GitLab