Merge branch 'move-generated-train-files' into 'master'

Move generated training files

See merge request !13

pom.xml

@@ -8,7 +8,7 @@
 
   <groupId>de.monticore.lang.monticar</groupId>
   <artifactId>cnnarch-gluon-generator</artifactId>
-  <version>0.1.5</version>
+  <version>0.1.6</version>
 
   <!-- == PROJECT DEPENDENCIES ============================================= -->
 

src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNArch2Gluon.java

@@ -47,6 +47,9 @@ public class CNNArch2Gluon extends CNNArch2MxNet {
         temp = archTc.process("CNNNet", Target.PYTHON);
         fileContentMap.put(temp.getKey(), temp.getValue());
 
+        temp = archTc.process("CNNDataLoader", Target.PYTHON);
+        fileContentMap.put(temp.getKey(), temp.getValue());
+
         temp = archTc.process("CNNCreator", Target.PYTHON);
         fileContentMap.put(temp.getKey(), temp.getValue());
 

src/main/java/de/monticore/lang/monticar/cnnarch/gluongenerator/CNNTrain2Gluon.java

@@ -21,7 +21,14 @@ public class CNNTrain2Gluon extends CNNTrain2MxNet {
         configDataList.add(configData);
         Map<String, Object> ftlContext = Collections.singletonMap("configurations", configDataList);
 
-        String templateContent = templateConfiguration.processTemplate(ftlContext, "CNNTrainer.ftl");
-        return Collections.singletonMap("CNNTrainer_" + getInstanceName() + ".py", templateContent);
+        Map<String, String> fileContentMap = new HashMap<>();
+
+        String cnnTrainTemplateContent = templateConfiguration.processTemplate(ftlContext, "CNNTrainer.ftl");
+        fileContentMap.put("CNNTrainer_" + getInstanceName() + ".py", cnnTrainTemplateContent);
+
+        String cnnSupervisedTrainerContent = templateConfiguration.processTemplate(ftlContext, "CNNSupervisedTrainer.ftl");
+        fileContentMap.put("supervised_trainer.py", cnnSupervisedTrainerContent);
+
+        return fileContentMap;
     }
 }
\ No newline at end of file

src/main/resources/templates/gluon/CNNCreator.ftl

 import mxnet as mx
 import logging
 import os
-import errno
-import shutil
-import h5py
-import sys
-import numpy as np
-import time
-from mxnet import gluon, autograd, nd
 from CNNNet_${tc.fullArchitectureName} import Net
 
-@mx.init.register
-class MyConstant(mx.init.Initializer):
-    def __init__(self, value):
-        super(MyConstant, self).__init__(value=value)
-        self.value = value
-    def _init_weight(self, _, arr):
-        arr[:] = mx.nd.array(self.value)
-
 class ${tc.fileNameWithoutEnding}:
-
-    _data_dir_ = "${tc.dataPath}/"
     _model_dir_ = "model/${tc.componentName}/"
     _model_prefix_ = "model"
-    _input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
     _input_shapes_ = [<#list tc.architecture.inputs as input>(${tc.join(input.definition.type.dimensions, ",")})</#list>]
-    _output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
 
     def __init__(self):
         self.weight_initializer = mx.init.Normal()
@@ -60,176 +41,6 @@ class ${tc.fileNameWithoutEnding}:
             return lastEpoch
 
 
-    def load_data(self, batch_size):
-        train_h5, test_h5 = self.load_h5_files()
-
-        data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
-        data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
-
-        train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
-                                       train_h5[self._output_names_[0]],
-                                       batch_size=batch_size,
-                                       data_name=self._input_names_[0],
-                                       label_name=self._output_names_[0])
-        test_iter = None
-        if test_h5 != None:
-            test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
-                                          test_h5[self._output_names_[0]],
-                                          batch_size=batch_size,
-                                          data_name=self._input_names_[0],
-                                          label_name=self._output_names_[0])
-        return train_iter, test_iter, data_mean, data_std
-
-    def load_h5_files(self):
-        train_h5 = None
-        test_h5 = None
-        train_path = self._data_dir_ + "train.h5"
-        test_path = self._data_dir_ + "test.h5"
-        if os.path.isfile(train_path):
-            train_h5 = h5py.File(train_path, 'r')
-            if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
-                logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
-                              + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
-                sys.exit(1)
-            test_iter = None
-            if os.path.isfile(test_path):
-                test_h5 = h5py.File(test_path, 'r')
-                if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
-                    logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
-                                  + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
-                    sys.exit(1)
-            else:
-                logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
-            return train_h5, test_h5
-        else:
-            logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
-            sys.exit(1)
-
-
-    def train(self, batch_size=64,
-              num_epoch=10,
-              eval_metric='acc',
-              optimizer='adam',
-              optimizer_params=(('learning_rate', 0.001),),
-              load_checkpoint=True,
-              context='gpu',
-              checkpoint_period=5,
-              normalize=True):
-        if context == 'gpu':
-            mx_context = mx.gpu()
-        elif context == 'cpu':
-            mx_context = mx.cpu()
-        else:
-            logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
-
-        if 'weight_decay' in optimizer_params:
-            optimizer_params['wd'] = optimizer_params['weight_decay']
-            del optimizer_params['weight_decay']
-        if 'learning_rate_decay' in optimizer_params:
-            min_learning_rate = 1e-08
-            if 'learning_rate_minimum' in optimizer_params:
-                min_learning_rate = optimizer_params['learning_rate_minimum']
-                del optimizer_params['learning_rate_minimum']
-            optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
-                                                   optimizer_params['step_size'],
-                                                   factor=optimizer_params['learning_rate_decay'],
-                                                   stop_factor_lr=min_learning_rate)
-            del optimizer_params['step_size']
-            del optimizer_params['learning_rate_decay']
-
-
-        train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
-        if self.net == None:
-            if normalize:
-                self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
-            else:
-                self.construct(context=mx_context)
-
-        begin_epoch = 0
-        if load_checkpoint:
-            begin_epoch = self.load(mx_context)
-        else:
-            if os.path.isdir(self._model_dir_):
-                shutil.rmtree(self._model_dir_)
-
-        try:
-            os.makedirs(self._model_dir_)
-        except OSError:
-            if not os.path.isdir(self._model_dir_):
-                raise
-
-        trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
-
-        if self.net.last_layer == 'softmax':
-            loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
-        elif self.net.last_layer == 'sigmoid':
-            loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
-        elif self.net.last_layer == 'linear':
-            loss_function = mx.gluon.loss.L2Loss()
-        else: # TODO: Change default?
-            loss_function = mx.gluon.loss.L2Loss()
-            logging.warning("Invalid last_layer, defaulting to L2 loss")
-
-        speed_period = 50
-        tic = None
-
-        for epoch in range(begin_epoch, begin_epoch + num_epoch):
-            train_iter.reset()
-            for batch_i, batch in enumerate(train_iter):
-                data = batch.data[0].as_in_context(mx_context)
-                label = batch.label[0].as_in_context(mx_context)
-                with autograd.record():
-                    output = self.net(data)
-                    loss = loss_function(output, label)
-
-                loss.backward()
-                trainer.step(batch_size)
-
-                if tic is None:
-                    tic = time.time()
-                else:
-                    if batch_i % speed_period == 0:
-                        try:
-                            speed = speed_period * batch_size / (time.time() - tic)
-                        except ZeroDivisionError:
-                            speed = float("inf")
-
-                        logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
-
-                        tic = time.time()
-
-            tic = None
-
-            train_iter.reset()
-            metric = mx.metric.create(eval_metric)
-            for batch_i, batch in enumerate(train_iter):
-                data = batch.data[0].as_in_context(mx_context)
-                label = batch.label[0].as_in_context(mx_context)
-                output = self.net(data)
-                predictions = mx.nd.argmax(output, axis=1)
-                metric.update(preds=predictions, labels=label)
-            train_metric_score = metric.get()[1]
-
-            test_iter.reset()
-            metric = mx.metric.create(eval_metric)
-            for batch_i, batch in enumerate(test_iter):
-                data = batch.data[0].as_in_context(mx_context)
-                label = batch.label[0].as_in_context(mx_context)
-                output = self.net(data)
-                predictions = mx.nd.argmax(output, axis=1)
-                metric.update(preds=predictions, labels=label)
-            test_metric_score = metric.get()[1]
-
-            logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
-
-            if (epoch - begin_epoch) % checkpoint_period == 0:
-                self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
-
-        self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
-            + str(num_epoch + begin_epoch).zfill(4) + '.params')
-        self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
-
-
     def construct(self, context, data_mean=None, data_std=None):
         self.net = Net(data_mean=data_mean, data_std=data_std)
         self.net.collect_params().initialize(self.weight_initializer, ctx=context)

src/main/resources/templates/gluon/CNNDataLoader.ftl

+import os
+import h5py
+import mxnet as mx
+import logging
+import sys
+
+class ${tc.fullArchitectureName}DataLoader:
+    _input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
+    _output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
+
+    def __init__(self):
+        self._data_dir = "${tc.dataPath}/"
+
+    def load_data(self, batch_size):
+        train_h5, test_h5 = self.load_h5_files()
+
+        data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
+        data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
+
+        train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
+                                       train_h5[self._output_names_[0]],
+                                       batch_size=batch_size,
+                                       data_name=self._input_names_[0],
+                                       label_name=self._output_names_[0])
+        test_iter = None
+        if test_h5 != None:
+            test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
+                                          test_h5[self._output_names_[0]],
+                                          batch_size=batch_size,
+                                          data_name=self._input_names_[0],
+                                          label_name=self._output_names_[0])
+        return train_iter, test_iter, data_mean, data_std
+
+    def load_h5_files(self):
+        train_h5 = None
+        test_h5 = None
+        train_path = self._data_dir + "train.h5"
+        test_path = self._data_dir + "test.h5"
+        if os.path.isfile(train_path):
+            train_h5 = h5py.File(train_path, 'r')
+            if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
+                logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+                              + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+                sys.exit(1)
+            test_iter = None
+            if os.path.isfile(test_path):
+                test_h5 = h5py.File(test_path, 'r')
+                if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
+                    logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+                                  + "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
+                    sys.exit(1)
+            else:
+                logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
+            return train_h5, test_h5
+        else:
+            logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
+            sys.exit(1)
\ No newline at end of file

src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl

+import mxnet as mx
+import logging
+import numpy as np
+import time
+import os
+import shutil
+from mxnet import gluon, autograd, nd
+
+class CNNSupervisedTrainer(object):
+    def __init__(self, data_loader, net_constructor, net=None):
+        self._data_loader = data_loader
+        self._net_creator = net_constructor
+        self._net = net
+
+    def train(self, batch_size=64,
+              num_epoch=10,
+              eval_metric='acc',
+              optimizer='adam',
+              optimizer_params=(('learning_rate', 0.001),),
+              load_checkpoint=True,
+              context='gpu',
+              checkpoint_period=5,
+              normalize=True):
+        if context == 'gpu':
+            mx_context = mx.gpu()
+        elif context == 'cpu':
+            mx_context = mx.cpu()
+        else:
+            logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
+
+        if 'weight_decay' in optimizer_params:
+            optimizer_params['wd'] = optimizer_params['weight_decay']
+            del optimizer_params['weight_decay']
+        if 'learning_rate_decay' in optimizer_params:
+            min_learning_rate = 1e-08
+            if 'learning_rate_minimum' in optimizer_params:
+                min_learning_rate = optimizer_params['learning_rate_minimum']
+                del optimizer_params['learning_rate_minimum']
+            optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
+                                                   optimizer_params['step_size'],
+                                                   factor=optimizer_params['learning_rate_decay'],
+                                                   stop_factor_lr=min_learning_rate)
+            del optimizer_params['step_size']
+            del optimizer_params['learning_rate_decay']
+
+
+        train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
+        if self._net is None:
+            if normalize:
+                self._net_creator.construct(
+                    context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
+            else:
+                self._net_creator.construct(context=mx_context)
+
+        begin_epoch = 0
+        if load_checkpoint:
+            begin_epoch = self._net_creator.load(mx_context)
+        else:
+            if os.path.isdir(self._net_creator._model_dir_):
+                shutil.rmtree(self._net_creator._model_dir_)
+
+        self._net = self._net_creator.net
+
+        try:
+            os.makedirs(self._net_creator._model_dir_)
+        except OSError:
+            if not os.path.isdir(self._net_creator._model_dir_):
+                raise
+
+        trainer = mx.gluon.Trainer(self._net.collect_params(), optimizer, optimizer_params)
+
+        if self._net.last_layer == 'softmax':
+            loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
+        elif self._net.last_layer == 'sigmoid':
+            loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
+        elif self._net.last_layer == 'linear':
+            loss_function = mx.gluon.loss.L2Loss()
+        else: # TODO: Change default?
+            loss_function = mx.gluon.loss.L2Loss()
+            logging.warning("Invalid last_layer, defaulting to L2 loss")
+
+        speed_period = 50
+        tic = None
+
+        for epoch in range(begin_epoch, begin_epoch + num_epoch):
+            train_iter.reset()
+            for batch_i, batch in enumerate(train_iter):
+                data = batch.data[0].as_in_context(mx_context)
+                label = batch.label[0].as_in_context(mx_context)
+                with autograd.record():
+                    output = self._net(data)
+                    loss = loss_function(output, label)
+
+                loss.backward()
+                trainer.step(batch_size)
+
+                if tic is None:
+                    tic = time.time()
+                else:
+                    if batch_i % speed_period == 0:
+                        try:
+                            speed = speed_period * batch_size / (time.time() - tic)
+                        except ZeroDivisionError:
+                            speed = float("inf")
+
+                        logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
+
+                        tic = time.time()
+
+            tic = None
+
+            train_iter.reset()
+            metric = mx.metric.create(eval_metric)
+            for batch_i, batch in enumerate(train_iter):
+                data = batch.data[0].as_in_context(mx_context)
+                label = batch.label[0].as_in_context(mx_context)
+                output = self._net(data)
+                predictions = mx.nd.argmax(output, axis=1)
+                metric.update(preds=predictions, labels=label)
+            train_metric_score = metric.get()[1]
+
+            test_iter.reset()
+            metric = mx.metric.create(eval_metric)
+            for batch_i, batch in enumerate(test_iter):
+                data = batch.data[0].as_in_context(mx_context)
+                label = batch.label[0].as_in_context(mx_context)
+                output = self._net(data)
+                predictions = mx.nd.argmax(output, axis=1)
+                metric.update(preds=predictions, labels=label)
+            test_metric_score = metric.get()[1]
+
+            logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
+
+            if (epoch - begin_epoch) % checkpoint_period == 0:
+                self._net.save_parameters(self.parameter_path() + '-' + str(epoch).zfill(4) + '.params')
+
+        self._net.save_parameters(self.parameter_path() + '-' + str(num_epoch + begin_epoch).zfill(4) + '.params')
+        self._net.export(self.parameter_path() + '_newest', epoch=0)
+
+    def parameter_path(self):
+        return self._net_creator._model_dir_ + self._net_creator._model_prefix_
\ No newline at end of file

src/main/resources/templates/gluon/CNNTrainer.ftl

 import logging
 import mxnet as mx
+import supervised_trainer
 <#list configurations as config>
 import CNNCreator_${config.instanceName}
+import CNNDataLoader_${config.instanceName}
 </#list>
 
 if __name__ == "__main__":
@@ -11,8 +13,12 @@ if __name__ == "__main__":
     logger.addHandler(handler)
 
 <#list configurations as config>
-    ${config.instanceName} = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
-    ${config.instanceName}.train(
+    ${config.instanceName}_creator = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
+    ${config.instanceName}_loader = CNNDataLoader_${config.instanceName}.${config.instanceName}DataLoader()
+    ${config.instanceName}_trainer = supervised_trainer.CNNSupervisedTrainer(${config.instanceName}_loader,
+        ${config.instanceName}_creator)
+
+    ${config.instanceName}_trainer.train(
 <#if (config.batchSize)??>
         batch_size=${config.batchSize},
 </#if>

src/test/java/de/monticore/lang/monticar/cnnarch/gluongenerator/GenerationTest.java

@@ -54,6 +54,7 @@ public class GenerationTest extends AbstractSymtabTest {
                 Arrays.asList(
                 "CNNCreator_CifarClassifierNetwork.py",
                 "CNNNet_CifarClassifierNetwork.py",
+                "CNNDataLoader_CifarClassifierNetwork.py",
                 "CNNPredictor_CifarClassifierNetwork.h",
                 "execute_CifarClassifierNetwork",
                 "CNNBufferFile.h"));
@@ -72,6 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
                 Arrays.asList(
                         "CNNCreator_Alexnet.py",
                         "CNNNet_Alexnet.py",
+                        "CNNDataLoader_Alexnet.py",
                         "CNNPredictor_Alexnet.h",
                         "execute_Alexnet"));
     }
@@ -89,6 +91,7 @@ public class GenerationTest extends AbstractSymtabTest {
                 Arrays.asList(
                         "CNNCreator_VGG16.py",
                         "CNNNet_VGG16.py",
+                        "CNNDataLoader_VGG16.py",
                         "CNNPredictor_VGG16.h",
                         "execute_VGG16"));
     }
@@ -130,7 +133,8 @@ public class GenerationTest extends AbstractSymtabTest {
                 Paths.get("./target/generated-sources-cnnarch"),
                 Paths.get("./src/test/resources/target_code"),
                 Arrays.asList(
-                        "CNNTrainer_fullConfig.py"));
+                        "CNNTrainer_fullConfig.py",
+                        "supervised_trainer.py"));
     }
 
     @Test
@@ -146,7 +150,8 @@ public class GenerationTest extends AbstractSymtabTest {
                 Paths.get("./target/generated-sources-cnnarch"),
                 Paths.get("./src/test/resources/target_code"),
                 Arrays.asList(
-                        "CNNTrainer_simpleConfig.py"));
+                        "CNNTrainer_simpleConfig.py",
+                        "supervised_trainer.py"));
     }
 
     @Test
@@ -161,7 +166,8 @@ public class GenerationTest extends AbstractSymtabTest {
                 Paths.get("./target/generated-sources-cnnarch"),
                 Paths.get("./src/test/resources/target_code"),
                 Arrays.asList(
-                        "CNNTrainer_emptyConfig.py"));
+                        "CNNTrainer_emptyConfig.py",
+                        "supervised_trainer.py"));
     }
 
 

src/test/resources/target_code/CNNCreator_Alexnet.py

 import mxnet as mx
 import logging
 import os
-import errno
-import shutil
-import h5py
-import sys
-import numpy as np
-import time
-from mxnet import gluon, autograd, nd
 from CNNNet_Alexnet import Net
 
-@mx.init.register
-class MyConstant(mx.init.Initializer):