Updated version numbers and fixed some tests

pom.xml

@@ -16,11 +16,11 @@
 
     <!-- .. SE-Libraries .................................................. -->
     <emadl.version>0.2.8-SNAPSHOT</emadl.version>
-    <CNNTrain.version>0.3.2-SNAPSHOT</CNNTrain.version>
-    <cnnarch-generator.version>0.0.1-SNAPSHOT</cnnarch-generator.version>
+    <CNNTrain.version>0.3.4-SNAPSHOT</CNNTrain.version>
+    <cnnarch-generator.version>0.0.2-SNAPSHOT</cnnarch-generator.version>
     <cnnarch-mxnet-generator.version>0.2.16-SNAPSHOT</cnnarch-mxnet-generator.version>
     <cnnarch-caffe2-generator.version>0.2.12-SNAPSHOT</cnnarch-caffe2-generator.version>
-    <cnnarch-gluon-generator.version>0.2.1-SNAPSHOT</cnnarch-gluon-generator.version>
+    <cnnarch-gluon-generator.version>0.2.2-SNAPSHOT</cnnarch-gluon-generator.version>
     <embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
 
         <!-- .. Libraries .................................................. -->

src/test/resources/target_code/CNNCreator_cifar10_cifar10Classifier_net.py

@@ -104,10 +104,83 @@ class CNNCreator_cifar10_cifar10Classifier_net:
             logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
             sys.exit(1)
 
+    def loss_function(self, loss, params):
+        label = mx.symbol.var(name=self._output_names_[0], )
+        prediction = self.module.symbol.get_children()[0]
+
+        margin = params['margin'] if 'margin' in params else 1.0
+        sparseLabel = params['sparse_label'] if 'sparse_label' in params else True
+
+        if loss == 'softmax_cross_entropy':
+            fromLogits = params['from_logits'] if 'from_logits' in params else False
+            if not fromLogits:
+                prediction = mx.symbol.log_softmax(data=prediction, axis=1)
+            if sparseLabel:
+                loss_func = mx.symbol.mean(-mx.symbol.pick(prediction, label, axis=-1, keepdims=True), axis=0, exclude=True)
+            else:
+                label = mx.symbol.reshape_like(label, prediction)
+                loss_func = mx.symbol.mean(-mx.symbol.sum(prediction * label, axis=-1, keepdims=True), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="softmax_cross_entropy")
+        elif loss == 'cross_entropy':
+            prediction = mx.symbol.log(prediction)
+            if sparseLabel:
+                loss_func = mx.symbol.mean(-mx.symbol.pick(prediction, label, axis=-1, keepdims=True), axis=0, exclude=True)
+            else:
+                label = mx.symbol.reshape_like(label, prediction)
+                loss_func = mx.symbol.mean(-mx.symbol.sum(prediction * label, axis=-1, keepdims=True), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="cross_entropy")
+        elif loss == 'sigmoid_binary_cross_entropy':
+            loss_func = mx.symbol.LogisticRegressionOutput(data=prediction, name=self.module.symbol.name)
+        elif loss == 'l1':
+            loss_func = mx.symbol.MAERegressionOutput(data=prediction, name=self.module.symbol.name)
+        elif loss == 'l2':
+            label = mx.symbol.reshape_like(label, prediction)
+            loss_func = mx.symbol.mean(mx.symbol.square((label - prediction) / 2), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="L2")
+        elif loss == 'huber':
+            rho = params['rho'] if 'rho' in params else 1
+            label = mx.symbol.reshape_like(label, prediction)
+            loss_func = mx.symbol.abs(label - prediction)
+            loss_func = mx.symbol.where(loss_func > rho, loss_func - 0.5 * rho, (0.5 / rho) * mx.symbol.square(loss_func))
+            loss_func = mx.symbol.mean(loss_func, axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="huber")
+        elif loss == 'hinge':
+            label = mx.symbol.reshape_like(label, prediction)
+            loss_func = mx.symbol.mean(mx.symbol.relu(margin - prediction * label), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="hinge")
+        elif loss == 'squared_hinge':
+            label = mx.symbol.reshape_like(label, prediction)
+            loss_func = mx.symbol.mean(mx.symbol.square(mx.symbol.relu(margin - prediction * label)), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="squared_hinge")
+        elif loss == 'logistic':
+            labelFormat = params['label_format'] if 'label_format' in params else 'signed'
+            if labelFormat not in ["binary", "signed"]:
+                logging.error("label_format can only be signed or binary")
+            label = mx.symbol.reshape_like(label, prediction)
+            if labelFormat == 'signed':
+                label = (label + 1.0)/2.0
+            loss_func = mx.symbol.relu(prediction) - prediction * label
+            loss_func = loss_func + mx.symbol.Activation(-mx.symbol.abs(prediction), act_type="softrelu")
+            loss_func = mx.symbol.MakeLoss(mx.symbol.mean(loss_func, 0, exclude=True), name="logistic")
+        elif loss == 'kullback_leibler':
+            fromLogits = params['from_logits'] if 'from_logits' in params else True
+            if not fromLogits:
+                prediction = mx.symbol.log_softmax(prediction, axis=1)
+            loss_func = mx.symbol.mean(label * (mx.symbol.log(label) - prediction), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="kullback_leibler")
+        elif loss == 'log_cosh':
+            loss_func = mx.symbol.mean(mx.symbol.log(mx.symbol.cosh(prediction - label)), axis=0, exclude=True)
+            loss_func = mx.symbol.MakeLoss(loss_func, name="log_cosh")
+        else:
+            logging.error("Invalid loss parameter.")
+
+        return loss_func
 
     def train(self, batch_size=64,
               num_epoch=10,
               eval_metric='acc',
+              loss ='softmax_cross_entropy',
+              loss_params={},
               optimizer='adam',
               optimizer_params=(('learning_rate', 0.001),),
               load_checkpoint=True,
@@ -136,7 +209,6 @@ class CNNCreator_cifar10_cifar10Classifier_net:
             del optimizer_params['step_size']
             del optimizer_params['learning_rate_decay']
 
-
         train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
         if self.module == None:
             if normalize:
@@ -144,6 +216,14 @@ class CNNCreator_cifar10_cifar10Classifier_net:
             else:
                 self.construct(mx_context)
 
+        loss_func = self.loss_function(loss=loss, params=loss_params)
+
+        self.module = mx.mod.Module(
+            symbol=mx.symbol.Group([loss_func, mx.symbol.BlockGrad(self.module.symbol.get_children()[0], name="pred")]),
+            data_names=self._input_names_,
+            label_names=self._output_names_,
+            context=mx_context)
+
         begin_epoch = 0
         if load_checkpoint:
             begin_epoch = self.load(mx_context)
@@ -157,9 +237,11 @@ class CNNCreator_cifar10_cifar10Classifier_net:
             if not os.path.isdir(self._model_dir_):
                 raise
 
+        metric = mx.metric.create(eval_metric, output_names=['pred_output'])
+
         self.module.fit(
             train_data=train_iter,
-            eval_metric=eval_metric,
+            eval_metric=metric,
             eval_data=test_iter,
             optimizer=optimizer,
             optimizer_params=optimizer_params,
@@ -656,8 +738,10 @@ class CNNCreator_cifar10_cifar10Classifier_net:
             num_hidden=10,
             no_bias=False,
             name="fc32_")
-
-        softmax = mx.symbol.SoftmaxOutput(data=fc32_,
+        softmax32_ = mx.symbol.softmax(data=fc32_,
+            axis=1,
+            name="softmax32_")
+        softmax = mx.symbol.SoftmaxOutput(data=softmax32_,
             name="softmax")
 
         self.module = mx.mod.Module(symbol=mx.symbol.Group([softmax]),

src/test/resources/target_code/gluon/CNNNet_mnist_mnistClassifier_net.py

@@ -120,7 +120,7 @@ class Net_0(gluon.HybridBlock):
             self.fc3_ = gluon.nn.Dense(units=10, use_bias=True)
             # fc3_, output shape: {[10,1,1]}
 
-        self.last_layers['predictions'] = 'softmax'
+            self.softmax3_ = Softmax()
 
 
     def hybrid_forward(self, F, image):
@@ -134,6 +134,7 @@ class Net_0(gluon.HybridBlock):
         fc2_ = self.fc2_(fc2_flatten_)
         relu2_ = self.relu2_(fc2_)
         fc3_ = self.fc3_(relu2_)
-        outputs.append(fc3_)
+        softmax3_ = self.softmax3_(fc3_)
+        outputs.append(softmax3_)
 
         return outputs[0]

src/test/resources/target_code/gluon/CNNSupervisedTrainer_mnist_mnistClassifier_net.py

@@ -6,6 +6,31 @@ import os
 import shutil
 from mxnet import gluon, autograd, nd
 
+class CrossEntropyLoss(gluon.loss.Loss):
+    def __init__(self, axis=-1, sparse_label=True, weight=None, batch_axis=0, **kwargs):
+        super(CrossEntropyLoss, self).__init__(weight, batch_axis, **kwargs)
+        self._axis = axis
+        self._sparse_label = sparse_label
+
+    def hybrid_forward(self, F, pred, label, sample_weight=None):
+        pred = F.log(pred)
+        if self._sparse_label:
+            loss = -F.pick(pred, label, axis=self._axis, keepdims=True)
+        else:
+            label = gluon.loss._reshape_like(F, label, pred)
+            loss = -F.sum(pred * label, axis=self._axis, keepdims=True)
+        loss = gluon.loss._apply_weighting(F, loss, self._weight, sample_weight)
+        return F.mean(loss, axis=self._batch_axis, exclude=True)
+
+class LogCoshLoss(gluon.loss.Loss):
+    def __init__(self, weight=None, batch_axis=0, **kwargs):
+        super(LogCoshLoss, self).__init__(weight, batch_axis, **kwargs)
+
+    def hybrid_forward(self, F, pred, label, sample_weight=None):
+        loss = F.log(F.cosh(pred - label))
+        loss = gluon.loss._apply_weighting(F, loss, self._weight, sample_weight)
+        return F.mean(loss, axis=self._batch_axis, exclude=True)
+
 class CNNSupervisedTrainer_mnist_mnistClassifier_net:
     def __init__(self, data_loader, net_constructor):
         self._data_loader = data_loader
@@ -15,6 +40,8 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
     def train(self, batch_size=64,
               num_epoch=10,
               eval_metric='acc',
+              loss ='softmax_cross_entropy',
+              loss_params={},
               optimizer='adam',
               optimizer_params=(('learning_rate', 0.001),),
               load_checkpoint=True,
@@ -68,19 +95,36 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
 
         trainers = [mx.gluon.Trainer(network.collect_params(), optimizer, optimizer_params) for network in self._networks.values()]
 
-        loss_functions = {}
-
-        for network in self._networks.values():
-            for output_name, last_layer in network.last_layers.items():
-                if last_layer == 'softmax':
-                    loss_functions[output_name] = mx.gluon.loss.SoftmaxCrossEntropyLoss()
-                elif last_layer == 'sigmoid':
-                    loss_functions[output_name] = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
-                elif last_layer == 'linear':
-                    loss_functions[output_name] = mx.gluon.loss.L2Loss()
-                else:
-                    loss_functions[output_name] = mx.gluon.loss.L2Loss()
-                    logging.warning("Invalid last layer, defaulting to L2 loss")
+        margin = loss_params['margin'] if 'margin' in loss_params else 1.0
+        sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
+        if loss == 'softmax_cross_entropy':
+            fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
+            loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
+        elif loss == 'sigmoid_binary_cross_entropy':
+            loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
+        elif loss == 'cross_entropy':
+            loss_function = CrossEntropyLoss(sparse_label=sparseLabel)
+        elif loss == 'l2':
+            loss_function = mx.gluon.loss.L2Loss()
+        elif loss == 'l1':
+            loss_function = mx.gluon.loss.L2Loss()
+        elif loss == 'huber':
+            rho = loss_params['rho'] if 'rho' in loss_params else 1
+            loss_function = mx.gluon.loss.HuberLoss(rho=rho)
+        elif loss == 'hinge':
+            loss_function = mx.gluon.loss.HingeLoss(margin=margin)
+        elif loss == 'squared_hinge':
+            loss_function = mx.gluon.loss.SquaredHingeLoss(margin=margin)
+        elif loss == 'logistic':
+            labelFormat = loss_params['label_format'] if 'label_format' in loss_params else 'signed'
+            loss_function = mx.gluon.loss.LogisticLoss(label_format=labelFormat)
+        elif loss == 'kullback_leibler':
+            fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else True
+            loss_function = mx.gluon.loss.KLDivLoss(from_logits=fromLogits)
+        elif loss == 'log_cosh':
+            loss_function = LogCoshLoss()
+        else:
+            logging.error("Invalid loss parameter.")
 
         speed_period = 50
         tic = None
@@ -95,7 +139,7 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
                     predictions_output = self._networks[0](image_data)
 
                     loss = \
-                        loss_functions['predictions'](predictions_output, predictions_label)
+                        loss_function(predictions_output, predictions_label)
 
                 loss.backward()