Merge branch 'process_loss_parameter' into 'master'

Processed loss parameter and set default value based on the last layer of the network architecture

See merge request !22

pom.xml

@@ -16,7 +16,7 @@
 
     <!-- .. SE-Libraries .................................................. -->
       <CNNArch.version>0.2.7</CNNArch.version>
-      <CNNTrain.version>0.2.5</CNNTrain.version>
+      <CNNTrain.version>0.2.6-SNAPSHOT</CNNTrain.version>
       <embedded-montiarc-math-generator>0.1.2-SNAPSHOT</embedded-montiarc-math-generator>
     
         <!-- .. Libraries .................................................. -->

src/main/java/de/monticore/lang/monticar/cnnarch/caffe2generator/CNNArchTemplateController.java

@@ -45,6 +45,10 @@ public class CNNArchTemplateController {
     private ArchitectureElementData dataElement;
 
 
+    public String loss;
+    public static final String CROSS_ENTROPY = "cross_entropy";
+    public static final String EUCLIDEAN = "euclidean";
+
     public CNNArchTemplateController(ArchitectureSymbol architecture) {
         setArchitecture(architecture);
     }
@@ -123,6 +127,10 @@ public class CNNArchTemplateController {
         return list;
     }
 
+    public String getArchitectureLoss(){
+        return this.loss;
+    }
+
     public void include(String relativePath, String templateWithoutFileEnding, Writer writer){
         String templatePath = relativePath + templateWithoutFileEnding + FTL_FILE_ENDING;
         Map<String, Object> ftlContext = new HashMap<>();
@@ -236,18 +244,30 @@ public class CNNArchTemplateController {
 
 
     public boolean isLogisticRegressionOutput(ArchitectureElementSymbol architectureElement){
-        return isTOutput(Sigmoid.class, architectureElement);
+        if (isTOutput(Sigmoid.class, architectureElement)){
+            this.loss = CROSS_ENTROPY;
+            return true;
+        }
+        return false;
     }
 
     public boolean isLinearRegressionOutput(ArchitectureElementSymbol architectureElement){
-        return architectureElement.isOutput()
+        if (architectureElement.isOutput()
                 && !isLogisticRegressionOutput(architectureElement)
-                && !isSoftmaxOutput(architectureElement);
+                && !isSoftmaxOutput(architectureElement)){
+            this.loss = EUCLIDEAN;
+            return true;
+        }
+        return false;
     }
 
 
     public boolean isSoftmaxOutput(ArchitectureElementSymbol architectureElement){
-        return isTOutput(Softmax.class, architectureElement);
+        if (isTOutput(Softmax.class, architectureElement)){
+            this.loss = CROSS_ENTROPY;
+            return true;
+        }
+        return false;
     }
 
     private boolean isTOutput(Class inputPredefinedLayerClass, ArchitectureElementSymbol architectureElement){

src/main/java/de/monticore/lang/monticar/cnnarch/caffe2generator/ConfigurationData.java

@@ -67,6 +67,13 @@ public class ConfigurationData {
         return getConfiguration().getEntry("eval_metric").getValue().toString();
     }
 
+    public String getLoss() {
+        if (!getConfiguration().getEntryMap().containsKey("loss")) {
+            return null;
+        }
+        return getConfiguration().getEntry("loss").getValue().toString();
+    }
+
     public String getOptimizerName() {
         if (getConfiguration().getOptimizer() == null) {
             return null;

src/main/resources/templates/caffe2/CNNCreator.ftl

@@ -65,10 +65,14 @@ class ${tc.fileNameWithoutEnding}:
 ${tc.include(tc.architecture.body)}
 
     # this adds the loss and optimizer
-    def add_training_operators(self, model, output, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) :
+    def add_training_operators(self, model, output, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) :
     	with core.DeviceScope(device_opts):
-    		xent = model.LabelCrossEntropy([output, label], 'xent')
-    		loss = model.AveragedLoss(xent, "loss")
+    		if loss == 'cross_entropy':
+    		    xent = model.LabelCrossEntropy([output, label], 'xent')
+    		    loss = model.AveragedLoss(xent, "loss")
+    		elif loss == 'euclidean':
+    		    dist = model.net.SquaredL2Distance([label, output], 'dist')
+    		    loss = dist.AveragedLoss([], ['loss'])
 
     		model.AddGradientOperators([loss])
 
@@ -105,7 +109,7 @@ ${tc.include(tc.architecture.body)}
                 accuracy = brew.accuracy(model, [output, label], "accuracy", top_k=3)
             return accuracy
 
-    def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9) :
+    def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', loss='${tc.architectureLoss}', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9) :
         if context == 'cpu':
             device_opts = core.DeviceOption(caffe2_pb2.CPU, 0)
             print("CPU mode selected")
@@ -120,7 +124,7 @@ ${tc.include(tc.architecture.body)}
     	train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
     	data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
     	${tc.join(tc.architectureOutputs, ",", "","")} = self.create_model(train_model, data, device_opts=device_opts, is_test=False)
-    	self.add_training_operators(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
+    	self.add_training_operators(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
     	self.add_accuracy(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, eval_metric)
     	with core.DeviceScope(device_opts):
     		brew.add_weight_decay(train_model, weight_decay)

src/main/resources/templates/caffe2/CNNTrainer.ftl

@@ -29,6 +29,9 @@ if __name__ == "__main__":
 <#if (config.evalMetric)??>
         eval_metric='${config.evalMetric}',
 </#if>
+<#if (config.loss)??>
+        loss='${config.loss}',
+</#if>
 <#if (config.configuration.optimizer)??>
         opt_type='${config.optimizerName}',
 <#list config.optimizerParams?keys as param>

src/test/resources/target_code/CNNCreator_LeNet.py

@@ -82,10 +82,14 @@ class CNNCreator_LeNet:
     		return predictions
 
     # this adds the loss and optimizer
-    def add_training_operators(self, model, output, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) :
+    def add_training_operators(self, model, output, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) :
     	with core.DeviceScope(device_opts):
-    		xent = model.LabelCrossEntropy([output, label], 'xent')
-    		loss = model.AveragedLoss(xent, "loss")
+    		if loss == 'cross_entropy':
+    		    xent = model.LabelCrossEntropy([output, label], 'xent')
+    		    loss = model.AveragedLoss(xent, "loss")
+    		elif loss == 'euclidean':
+    		    dist = model.net.SquaredL2Distance([label, output], 'dist')
+    		    loss = dist.AveragedLoss([], ['loss'])
 
     		model.AddGradientOperators([loss])
 
@@ -122,7 +126,7 @@ class CNNCreator_LeNet:
                 accuracy = brew.accuracy(model, [output, label], "accuracy", top_k=3)
             return accuracy
 
-    def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9) :
+    def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', loss='cross_entropy', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9) :
         if context == 'cpu':
             device_opts = core.DeviceOption(caffe2_pb2.CPU, 0)
             print("CPU mode selected")
@@ -137,7 +141,7 @@ class CNNCreator_LeNet:
     	train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
     	data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
     	predictions = self.create_model(train_model, data, device_opts=device_opts, is_test=False)
-    	self.add_training_operators(train_model, predictions, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
+    	self.add_training_operators(train_model, predictions, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
     	self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
     	with core.DeviceScope(device_opts):
     		brew.add_weight_decay(train_model, weight_decay)

src/test/resources/target_code/CNNCreator_VGG16.py

@@ -128,10 +128,14 @@ class CNNCreator_VGG16:
     		return predictions
 
     # this adds the loss and optimizer
-    def add_training_operators(self, model, output, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) :
+    def add_training_operators(self, model, output, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) :
     	with core.DeviceScope(device_opts):
-    		xent = model.LabelCrossEntropy([output, label], 'xent')
-    		loss = model.AveragedLoss(xent, "loss")
+    		if loss == 'cross_entropy':
+    		    xent = model.LabelCrossEntropy([output, label], 'xent')
+    		    loss = model.AveragedLoss(xent, "loss")
+    		elif loss == 'euclidean':
+    		    dist = model.net.SquaredL2Distance([label, output], 'dist')
+    		    loss = dist.AveragedLoss([], ['loss'])
 
     		model.AddGradientOperators([loss])
 
@@ -168,7 +172,7 @@ class CNNCreator_VGG16:
                 accuracy = brew.accuracy(model, [output, label], "accuracy", top_k=3)
             return accuracy
 
-    def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9) :
+    def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', loss='cross_entropy', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9) :
         if context == 'cpu':
             device_opts = core.DeviceOption(caffe2_pb2.CPU, 0)
             print("CPU mode selected")
@@ -183,7 +187,7 @@ class CNNCreator_VGG16:
     	train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
     	data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
     	predictions = self.create_model(train_model, data, device_opts=device_opts, is_test=False)
-    	self.add_training_operators(train_model, predictions, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
+    	self.add_training_operators(train_model, predictions, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
     	self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
     	with core.DeviceScope(device_opts):
     		brew.add_weight_decay(train_model, weight_decay)

src/test/resources/target_code/CNNTrainer_fullConfig.py

@@ -18,6 +18,7 @@ if __name__ == "__main__":
         batch_size=100,
         context='gpu',
         eval_metric='mse',
+        loss='cross_entropy',
         opt_type='rmsprop',
         epsilon=1.0E-6,
         weight_decay=0.01,

src/test/resources/valid_tests/FullConfig.cnnt

@@ -3,6 +3,7 @@ configuration FullConfig{
      batch_size : 100
      load_checkpoint : true
      eval_metric : mse
+     loss : cross_entropy
      context : gpu
      normalize : true
      optimizer : rmsprop{