Reenabled Gluon GenerationTest

Former-commit-id: 64148a83

src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java

@@ -171,7 +171,7 @@ public class GenerationTest extends AbstractSymtabTest {
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
 
-        /*checkFilesAreEqual(
+        checkFilesAreEqual(
                 Paths.get("./target/generated-sources-emadl"),
                 Paths.get("./src/test/resources/target_code/gluon"),
                 Arrays.asList(
@@ -188,7 +188,7 @@ public class GenerationTest extends AbstractSymtabTest {
                         "CNNTranslator.h",
                         "mnist_mnistClassifier_calculateClass.h",
                         "CNNTrainer_mnist_mnistClassifier_net.py",
-                        "mnist_mnistClassifier_net.h"));*/
+                        "mnist_mnistClassifier_net.h"));
     }
 
     @Test

src/test/java/de/monticore/lang/monticar/emadl/IntegrationPythonWrapperTest.java

@@ -13,9 +13,6 @@ import java.util.stream.Collectors;
 import static junit.framework.TestCase.assertTrue;
 import static org.junit.Assume.assumeFalse;
 
-/**
- *
- */
 public class IntegrationPythonWrapperTest extends AbstractSymtabTest {
     @Test
     public void testGluonReinforcementModelRosEnvironment() {

src/test/resources/target_code/gluon/CNNCreator_mnist_mnistClassifier_net.py

 import mxnet as mx
 import logging
 import os
+
 from CNNNet_mnist_mnistClassifier_net import Net_0
 
 class CNNCreator_mnist_mnistClassifier_net:
@@ -50,7 +51,7 @@ class CNNCreator_mnist_mnistClassifier_net:
         self.networks[0] = Net_0(data_mean=data_mean, data_std=data_std)
         self.networks[0].collect_params().initialize(self.weight_initializer, ctx=context)
         self.networks[0].hybridize()
-        self.networks[0](mx.nd.zeros((1,1,28,28,), ctx=context))
+        self.networks[0](mx.nd.zeros((1, 1,28,28,), ctx=context))
 
         if not os.path.exists(self._model_dir_):
             os.makedirs(self._model_dir_)

src/test/resources/target_code/gluon/CNNDataLoader_mnist_mnistClassifier_net.py

@@ -12,42 +12,58 @@ class CNNDataLoader_mnist_mnistClassifier_net:
     def __init__(self):
         self._data_dir = "data/mnist.LeNetNetwork/"
 
-    def load_data(self, batch_size):
+    def load_data(self, train_batch_size, test_batch_size):
         train_h5, test_h5 = self.load_h5_files()
 
         train_data = {}
         data_mean = {}
         data_std = {}
+        train_images = {}
 
         for input_name in self._input_names_:
             train_data[input_name] = train_h5[input_name]
             data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
             data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
 
+            if 'images' in train_h5:
+                train_images = train_h5['images']
+
         train_label = {}
+        index = 0
         for output_name in self._output_names_:
-            train_label[output_name] = train_h5[output_name]
+            train_label[index] = train_h5[output_name]
+            index += 1
 
         train_iter = mx.io.NDArrayIter(data=train_data,
                                        label=train_label,
-                                       batch_size=batch_size)
+                                       batch_size=train_batch_size)
+
+        train_test_iter = mx.io.NDArrayIter(data=train_data,
+                                            label=train_label,
+                                            batch_size=test_batch_size)
 
         test_iter = None
 
         if test_h5 != None:
             test_data = {}
+            test_images = {}
             for input_name in self._input_names_:
                 test_data[input_name] = test_h5[input_name]
 
+                if 'images' in test_h5:
+                    test_images = test_h5['images']
+
             test_label = {}
+            index = 0
             for output_name in self._output_names_:
-                test_label[output_name] = test_h5[output_name]
+                test_label[index] = test_h5[output_name]
+                index += 1
 
             test_iter = mx.io.NDArrayIter(data=test_data,
                                           label=test_label,
-                                          batch_size=batch_size)
+                                          batch_size=test_batch_size)
 
-        return train_iter, test_iter, data_mean, data_std
+        return train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images
 
     def load_h5_files(self):
         train_h5 = None

src/test/resources/target_code/gluon/CNNNet_mnist_mnistClassifier_net.py

@@ -2,53 +2,15 @@ import mxnet as mx
 import numpy as np
 from mxnet import gluon
 
-class OneHot(gluon.HybridBlock):
-    def __init__(self, size, **kwargs):
-        super(OneHot, self).__init__(**kwargs)
-        with self.name_scope():
-            self.size = size
-
-    def hybrid_forward(self, F, x):
-        return F.one_hot(indices=F.argmax(data=x, axis=1), depth=self.size)
-
-
-class Softmax(gluon.HybridBlock):
-    def __init__(self, **kwargs):
-        super(Softmax, self).__init__(**kwargs)
-
-    def hybrid_forward(self, F, x):
-        return F.softmax(x)
-
-
-class Split(gluon.HybridBlock):
-    def __init__(self, num_outputs, axis=1, **kwargs):
-        super(Split, self).__init__(**kwargs)
-        with self.name_scope():
-            self.axis = axis
-            self.num_outputs = num_outputs
-
-    def hybrid_forward(self, F, x):
-        return F.split(data=x, axis=self.axis, num_outputs=self.num_outputs)
-
-
-class Concatenate(gluon.HybridBlock):
-    def __init__(self, dim=1, **kwargs):
-        super(Concatenate, self).__init__(**kwargs)
-        with self.name_scope():
-            self.dim = dim
-
-    def hybrid_forward(self, F, *x):
-        return F.concat(*x, dim=self.dim)
-
 
 class ZScoreNormalization(gluon.HybridBlock):
     def __init__(self, data_mean, data_std, **kwargs):
         super(ZScoreNormalization, self).__init__(**kwargs)
         with self.name_scope():
             self.data_mean = self.params.get('data_mean', shape=data_mean.shape,
-                                             init=mx.init.Constant(data_mean.asnumpy().tolist()), differentiable=False)
+                init=mx.init.Constant(data_mean.asnumpy().tolist()), differentiable=False)
             self.data_std = self.params.get('data_std', shape=data_mean.shape,
-                                            init=mx.init.Constant(data_std.asnumpy().tolist()), differentiable=False)
+                init=mx.init.Constant(data_std.asnumpy().tolist()), differentiable=False)
 
     def hybrid_forward(self, F, x, data_mean, data_std):
         x = F.broadcast_sub(x, data_mean)
@@ -64,9 +26,9 @@ class Padding(gluon.HybridBlock):
 
     def hybrid_forward(self, F, x):
         x = F.pad(data=x,
-                  mode='constant',
-                  pad_width=self.pad_width,
-                  constant_value=0)
+            mode='constant',
+            pad_width=self.pad_width,
+            constant_value=0)
         return x
 
 
@@ -78,6 +40,52 @@ class NoNormalization(gluon.HybridBlock):
         return x
 
 
+class Reshape(gluon.HybridBlock):
+    def __init__(self, shape, **kwargs):
+        super(Reshape, self).__init__(**kwargs)
+        with self.name_scope():
+            self.shape = shape
+
+    def hybrid_forward(self, F, x):
+        return F.reshape(data=x, shape=self.shape)
+
+
+class CustomRNN(gluon.HybridBlock):
+    def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
+        super(CustomRNN, self).__init__(**kwargs)
+        with self.name_scope():
+            self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers,
+                                     bidirectional=bidirectional, activation='tanh', layout='NTC')
+
+    def hybrid_forward(self, F, data, state0):
+        output, [state0] = self.rnn(data, [F.swapaxes(state0, 0, 1)])
+        return output, F.swapaxes(state0, 0, 1)
+
+
+class CustomLSTM(gluon.HybridBlock):
+    def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
+        super(CustomLSTM, self).__init__(**kwargs)
+        with self.name_scope():
+            self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers,
+                                       bidirectional=bidirectional, layout='NTC')
+
+    def hybrid_forward(self, F, data, state0, state1):
+        output, [state0, state1] = self.lstm(data, [F.swapaxes(state0, 0, 1), F.swapaxes(state1, 0, 1)])
+        return output, F.swapaxes(state0, 0, 1), F.swapaxes(state1, 0, 1)
+
+
+class CustomGRU(gluon.HybridBlock):
+    def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
+        super(CustomGRU, self).__init__(**kwargs)
+        with self.name_scope():
+            self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers,
+                                     bidirectional=bidirectional, layout='NTC')
+
+    def hybrid_forward(self, F, data, state0):
+        output, [state0] = self.gru(data, [F.swapaxes(state0, 0, 1)])
+        return output, F.swapaxes(state0, 0, 1)
+
+
 class Net_0(gluon.HybridBlock):
     def __init__(self, data_mean=None, data_std=None, **kwargs):
         super(Net_0, self).__init__(**kwargs)
@@ -85,23 +93,16 @@ class Net_0(gluon.HybridBlock):
             if data_mean:
                 assert(data_std)
                 self.input_normalization_image_ = ZScoreNormalization(data_mean=data_mean['image_'],
-                                                                      data_std=data_std['image_'])
+                                                                               data_std=data_std['image_'])
             else:
                 self.input_normalization_image_ = NoNormalization()
 
             self.conv1_padding = Padding(padding=(0,0,0,0,2,2,2,2))
             self.conv1_ = gluon.nn.Conv2D(channels=20,
-<<<<<<< HEAD
                 kernel_size=(5,5),
                 strides=(1,1),
                 use_bias=True)
             # conv1_, output shape: {[20,28,28]}
-=======
-                                          kernel_size=(5,5),
-                                          strides=(1,1),
-                                          use_bias=True)
-            # conv1_, output shape: {[20,24,24]}
->>>>>>> rnn
 
             self.pool1_ = gluon.nn.MaxPool2D(
                 pool_size=(2,2),
@@ -110,17 +111,10 @@ class Net_0(gluon.HybridBlock):
 
             self.conv2_padding = Padding(padding=(0,0,0,0,2,2,2,2))
             self.conv2_ = gluon.nn.Conv2D(channels=50,
-<<<<<<< HEAD
                 kernel_size=(5,5),
                 strides=(1,1),
                 use_bias=True)
             # conv2_, output shape: {[50,14,14]}
-=======
-                                          kernel_size=(5,5),
-                                          strides=(1,1),
-                                          use_bias=True)
-            # conv2_, output shape: {[50,8,8]}
->>>>>>> rnn
 
             self.pool2_ = gluon.nn.MaxPool2D(
                 pool_size=(2,2),
@@ -134,8 +128,8 @@ class Net_0(gluon.HybridBlock):
             self.fc3_ = gluon.nn.Dense(units=10, use_bias=True, flatten=True)
             # fc3_, output shape: {[10,1,1]}
 
-            self.softmax3_ = Softmax()
 
+            pass
 
     def hybrid_forward(self, F, image_):
         image_ = self.input_normalization_image_(image_)
@@ -148,9 +142,8 @@ class Net_0(gluon.HybridBlock):
         fc2_ = self.fc2_(pool2_)
         relu2_ = self.relu2_(fc2_)
         fc3_ = self.fc3_(relu2_)
-        softmax3_ = self.softmax3_(fc3_)
+        softmax3_ = F.softmax(fc3_, axis=-1)
         predictions_ = softmax3_
 
         return predictions_
 
-

src/test/resources/target_code/gluon/CNNPredictor_mnist_mnistClassifier_net.h

@@ -16,7 +16,7 @@ public:
     const std::vector<std::string> input_keys = {
         "data"
     };
-    const std::vector<std::vector<mx_uint>> input_shapes = {{1, 1, 28, 28}};
+    const std::vector<std::vector<mx_uint>> input_shapes = {{1, 28, 28}};
     const bool use_gpu = false;
 
     PredictorHandle handle;

src/test/resources/target_code/gluon/CNNSupervisedTrainer_mnist_mnistClassifier_net.py

@@ -4,6 +4,9 @@ import numpy as np
 import time
 import os
 import shutil
+import pickle
+import math
+import sys
 from mxnet import gluon, autograd, nd
 
 class CrossEntropyLoss(gluon.loss.Loss):
@@ -31,6 +34,142 @@ class LogCoshLoss(gluon.loss.Loss):
         loss = gluon.loss._apply_weighting(F, loss, self._weight, sample_weight)
         return F.mean(loss, axis=self._batch_axis, exclude=True)
 
+class SoftmaxCrossEntropyLossIgnoreIndices(gluon.loss.Loss):
+    def __init__(self, axis=-1, ignore_indices=[], sparse_label=True, from_logits=False, weight=None, batch_axis=0, **kwargs):
+        super(SoftmaxCrossEntropyLossIgnoreIndices, self).__init__(weight, batch_axis, **kwargs)
+        self._axis = axis
+        self._ignore_indices = ignore_indices
+        self._sparse_label = sparse_label
+        self._from_logits = from_logits
+
+    def hybrid_forward(self, F, pred, label, sample_weight=None):
+        log_softmax = F.log_softmax
+        pick = F.pick
+        if not self._from_logits:
+            pred = log_softmax(pred, self._axis)
+        if self._sparse_label:
+            loss = -pick(pred, label, axis=self._axis, keepdims=True)
+        else:
+            label = _reshape_like(F, label, pred)
+            loss = -(pred * label).sum(axis=self._axis, keepdims=True)
+        #loss = _apply_weighting(F, loss, self._weight, sample_weight)
+        # ignore some indices for loss, e.g. <pad> tokens in NLP applications
+        for i in self._ignore_indices:
+            loss = loss * mx.nd.logical_not(mx.nd.equal(mx.nd.argmax(pred, axis=1), mx.nd.ones_like(mx.nd.argmax(pred, axis=1))*i))
+        return loss.mean(axis=self._batch_axis, exclude=True)
+
+@mx.metric.register
+class BLEU(mx.metric.EvalMetric):
+    N = 4
+
+    def __init__(self, exclude=None, name='bleu', output_names=None, label_names=None):
+        super(BLEU, self).__init__(name=name, output_names=output_names, label_names=label_names)
+
+        self._exclude = exclude or []
+
+        self._match_counts = [0 for _ in range(self.N)]
+        self._counts = [0 for _ in range(self.N)]
+
+        self._size_ref = 0
+        self._size_hyp = 0
+
+    def update(self, labels, preds):
+        labels, preds = mx.metric.check_label_shapes(labels, preds, True)
+
+        new_labels = self._convert(labels)
+        new_preds = self._convert(preds)
+
+        for label, pred in zip(new_labels, new_preds):
+            reference = [word for word in label if word not in self._exclude]
+            hypothesis = [word for word in pred if word not in self._exclude]
+
+            self._size_ref += len(reference)
+            self._size_hyp += len(hypothesis)
+
+            for n in range(self.N):
+                reference_ngrams = self._get_ngrams(reference, n + 1)
+                hypothesis_ngrams = self._get_ngrams(hypothesis, n + 1)
+
+                match_count = 0
+
+                for ngram in hypothesis_ngrams:
+                    if ngram in reference_ngrams:
+                        reference_ngrams.remove(ngram)
+
+                        match_count += 1
+
+                self._match_counts[n] += match_count
+                self._counts[n] += len(hypothesis_ngrams)
+
+    def get(self):
+        precisions = [sys.float_info.min for n in range(self.N)]
+
+        i = 1
+
+        for n in range(self.N):
+            match_counts = self._match_counts[n]
+            counts = self._counts[n]
+
+            if counts != 0:
+                if match_counts == 0:
+                    i *= 2
+                    match_counts = 1 / i
+
+                precisions[n] = match_counts / counts
+
+        bleu = self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
+
+        return (self.name, bleu)
+
+    def calculate(self):
+        precisions = [sys.float_info.min for n in range(self.N)]
+
+        i = 1
+
+        for n in range(self.N):
+            match_counts = self._match_counts[n]
+            counts = self._counts[n]
+
+            if counts != 0:
+                if match_counts == 0:
+                    i *= 2
+                    match_counts = 1 / i
+
+                precisions[n] = match_counts / counts
+
+        return self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
+
+    def _get_brevity_penalty(self):
+        if self._size_hyp >= self._size_ref:
+            return 1
+        else:
+            return math.exp(1 - (self._size_ref / self._size_hyp))
+
+    @staticmethod
+    def _get_ngrams(sentence, n):
+        ngrams = []
+
+        if len(sentence) >= n:
+            for i in range(len(sentence) - n + 1):
+                ngrams.append(sentence[i:i+n])
+
+        return ngrams
+
+    @staticmethod
+    def _convert(nd_list):
+        if len(nd_list) == 0:
+            return []
+
+        new_list = [[] for _ in range(nd_list[0].shape[0])]
+
+        for element in nd_list:
+            for i in range(element.shape[0]):
+                new_list[i].append(element[i].asscalar())
+
+        return new_list
+
+
+
 class CNNSupervisedTrainer_mnist_mnistClassifier_net:
     def __init__(self, data_loader, net_constructor):
         self._data_loader = data_loader
@@ -48,6 +187,7 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
               load_checkpoint=True,
               context='gpu',
               checkpoint_period=5,
+              save_attention_image=False,
               normalize=True):
         if context == 'gpu':
             mx_context = mx.gpu()
@@ -65,14 +205,16 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
                 min_learning_rate = optimizer_params['learning_rate_minimum']
                 del optimizer_params['learning_rate_minimum']
             optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
-                optimizer_params['step_size'],
-                factor=optimizer_params['learning_rate_decay'],
-                stop_factor_lr=min_learning_rate)
+                                                   optimizer_params['step_size'],
+                                                   factor=optimizer_params['learning_rate_decay'],
+                                                   stop_factor_lr=min_learning_rate)
             del optimizer_params['step_size']
             del optimizer_params['learning_rate_decay']
 
+        train_batch_size = batch_size
+        test_batch_size = batch_size
 
-        train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
+        train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(train_batch_size, test_batch_size)
 
         if normalize:
             self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
@@ -94,13 +236,17 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
             if not os.path.isdir(self._net_creator._model_dir_):
                 raise
 
-        trainers = [mx.gluon.Trainer(network.collect_params(), optimizer, optimizer_params) for network in self._networks.values()]
+        trainers = [mx.gluon.Trainer(network.collect_params(), optimizer, optimizer_params) for network in self._networks.values() if len(network.collect_params().values()) != 0]
 
         margin = loss_params['margin'] if 'margin' in loss_params else 1.0
         sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
+        #if loss == 'softmax_cross_entropy':
+        #    fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
+        #    loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
         if loss == 'softmax_cross_entropy':
             fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
-            loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
+            ignore_indices = [2]
+            loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
         elif loss == 'sigmoid_binary_cross_entropy':
             loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
         elif loss == 'cross_entropy':
@@ -133,17 +279,23 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
         for epoch in range(begin_epoch, begin_epoch + num_epoch):
             train_iter.reset()
             for batch_i, batch in enumerate(train_iter):
-                image_ = batch.data[0].as_in_context(mx_context)
-                predictions_label = batch.label[0].as_in_context(mx_context)
-
                 with autograd.record():
-                    predictions_ = mx.nd.zeros((batch_size, 10,), ctx=mx_context)
+                    labels = [batch.label[i].as_in_context(mx_context) for i in range(1)]
+
+                    image_ = batch.data[0].as_in_context(mx_context)
+
+                    predictions_ = mx.nd.zeros((train_batch_size, 10,), ctx=mx_context)
+
+
+                    lossList = []
 
                     predictions_ = self._networks[0](image_)
 
+                    lossList.append(loss_function(predictions_, labels[0]))
 
-                    loss = \
-                        loss_function(predictions_, predictions_label)
+                    loss = 0
+                    for element in lossList:
+                        loss = loss + element
 
                 loss.backward()
 
@@ -165,64 +317,117 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
 
             tic = None
 
-            train_iter.reset()
-            metric = mx.metric.create(eval_metric)
-            for batch_i, batch in enumerate(train_iter):
-                image_ = batch.data[0].as_in_context(mx_context)
+            train_test_iter.reset()
+            metric = mx.metric.create(eval_metric, **eval_metric_params)
+            for batch_i, batch in enumerate(train_test_iter):
+                if True:
+                    labels = [batch.label[i].as_in_context(mx_context) for i in range(1)]
 
-                labels = [
-                    batch.label[0].as_in_context(mx_context)
-                ]
+                    image_ = batch.data[0].as_in_context(mx_context)
+
+                    predictions_ = mx.nd.zeros((test_batch_size, 10,), ctx=mx_context)
 
-<<<<<<< HEAD