Adapt different backends, moved to a newer EMADL version

.gitlab-ci.yml

@@ -28,20 +28,23 @@ masterJobLinux:
   image: maven:3-jdk-8
   script: 
   - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean deploy --settings settings.xml
+  - cat target/site/jacoco/index.html
+  - mvn package sonar:sonar -s settings.xml
   only:
   - master
   
-masterJobWindows:
-  stage: windows
-  script: 
-  - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
-  tags:
-  - Windows10
+#masterJobWindows:
+#  stage: windows
+#  script:
+#  - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
+#  tags:
+#  - Windows10
 
 BranchJobLinux:
   stage: linux
   image: maven:3-jdk-8
   script: 
   - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
+  - cat target/site/jacoco/index.html
   except:
   - master

README.md

+![pipeline](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/build.svg)
+![coverage](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/coverage.svg)
 # EMADL2CPP
 Generates CPP/Python code for EmbeddedMontiArcDL.
 See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/EMADL-Demo) for more information on how the generated code can be used.
\ No newline at end of file

pom.xml

@@ -8,17 +8,18 @@
 
   <groupId>de.monticore.lang.monticar</groupId>
   <artifactId>embedded-montiarc-emadl-generator</artifactId>
-  <version>0.2.1-SNAPSHOT</version>
+  <version>0.2.2-SNAPSHOT</version>
 
   <!-- == PROJECT DEPENDENCIES ============================================= -->
 
   <properties>
 
     <!-- .. SE-Libraries .................................................. -->
-    <emadl.version>0.2.1-SNAPSHOT</emadl.version>
-    <CNNTrain.version>0.2.1-SNAPSHOT</CNNTrain.version>
-    <cnnarch-generator.version>0.2.1-SNAPSHOT</cnnarch-generator.version>
-    <embedded-montiarc-math-generator>0.0.10</embedded-montiarc-math-generator>
+    <emadl.version>0.2.2-SNAPSHOT</emadl.version>
+    <CNNTrain.version>0.2.4-SNAPSHOT</CNNTrain.version>
+    <cnnarch-mxnet-generator.version>0.2.3-SNAPSHOT</cnnarch-mxnet-generator.version>
+    <cnnarch-caffe2-generator.version>0.2.2-SNAPSHOT</cnnarch-caffe2-generator.version>
+    <embedded-montiarc-math-generator>0.0.25-SNAPSHOT</embedded-montiarc-math-generator>
     
         <!-- .. Libraries .................................................. -->
         <guava.version>18.0</guava.version>
@@ -31,6 +32,7 @@
         <compiler.plugin>3.3</compiler.plugin>
         <source.plugin>2.4</source.plugin>
         <shade.plugin>2.4.3</shade.plugin>
+        <jacoco.plugin>0.8.1</jacoco.plugin>
 
     <!-- Classifiers -->
     <grammars.classifier>grammars</grammars.classifier>
@@ -68,7 +70,13 @@
         <dependency>
             <groupId>de.monticore.lang.monticar</groupId>
             <artifactId>cnnarch-mxnet-generator</artifactId>
-            <version>${cnnarch-generator.version}</version>
+            <version>${cnnarch-mxnet-generator.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>de.monticore.lang.monticar</groupId>
+            <artifactId>cnnarch-caffe2-generator</artifactId>
+            <version>${cnnarch-caffe2-generator.version}</version>
         </dependency>
 
         <dependency>
@@ -132,6 +140,27 @@
         <version>2.8.1</version>
       </plugin>
 
+        <!-- Test coverage -->
+        <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.plugin}</version>
+            <executions>
+                <execution>
+                    <id>pre-unit-test</id>
+                    <goals>
+                        <goal>prepare-agent</goal>
+                    </goals>
+                </execution>
+                <execution>
+                    <id>post-unit-test</id>
+                    <phase>test</phase>
+                    <goals>
+                        <goal>report</goal>
+                    </goals>
+                </execution>
+            </executions>
+        </plugin>
       <!-- Other Configuration -->
       <plugin>
         <artifactId>maven-compiler-plugin</artifactId>

settings.xml

@@ -57,6 +57,24 @@
   </mirrors>
 
   <profiles>
+    <profile>
+      <id>sonar</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <properties>
+        <!-- Optional URL to server. Default value is http://localhost:9000 -->
+        <sonar.host.url>
+          https://metric.se.rwth-aachen.de
+        </sonar.host.url>
+        <sonar.login>
+          jenkins
+        </sonar.login>
+        <sonar.password>
+          ${env.sonar}
+        </sonar.password>
+      </properties>
+    </profile>
 
     <profile>
       <id>se-nexus</id>

src/main/java/de/monticore/lang/monticar/emadl/generator/Backend.java

+package de.monticore.lang.monticar.emadl.generator;
+
+
+import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
+import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
+import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
+
+import java.util.Optional;
+
+public enum Backend {
+    MXNET{
+        @Override
+        public CNNArchGenerator getGenerator() {
+            return new CNNArch2MxNet();
+        }
+    },
+    CAFFE2{
+        @Override
+        public CNNArchGenerator getGenerator() {
+            return new CNNArch2Caffe2();
+        }
+    };
+
+    public abstract CNNArchGenerator getGenerator();
+
+    public static Optional<Backend> getBackendFromString(String backend){
+        switch (backend){
+            case "MXNET":
+                return Optional.of(MXNET);
+
+            case "CAFFE2":
+                return Optional.of(CAFFE2);
+
+            default:
+                return Optional.empty();
+        }
+    }
+}

src/main/java/de/monticore/lang/monticar/emadl/generator/AbstractSymtab.java → src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLAbstractSymtab.java

@@ -44,7 +44,10 @@ import de.monticore.symboltable.Scope;
 import java.nio.file.Paths;
 import java.util.Arrays;
 
-public class AbstractSymtab {
+public class EMADLAbstractSymtab {
+    public EMADLAbstractSymtab() {
+    }
+
     public static TaggingResolver createSymTabAndTaggingResolver(String... modelPath) {
         Scope scope = createSymTab(modelPath);
         TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGeneratorCli.java

@@ -26,6 +26,7 @@ import freemarker.template.TemplateException;
 import org.apache.commons.cli.*;
 
 import java.io.IOException;
+import java.util.Optional;
 
 public class EMADLGeneratorCli {
 
@@ -49,6 +50,12 @@ public class EMADLGeneratorCli {
             .hasArg(true)
             .required(false)
             .build();
+    public static final Option OPTION_BACKEND = Option.builder("b")
+            .longOpt("backend")
+            .desc("deep-learning-framework backend. Options: MXNET, CAFFE2")
+            .hasArg(true)
+            .required(false)
+            .build();
 
     private EMADLGeneratorCli() {
     }
@@ -67,6 +74,7 @@ public class EMADLGeneratorCli {
         options.addOption(OPTION_MODELS_PATH);
         options.addOption(OPTION_ROOT_MODEL);
         options.addOption(OPTION_OUTPUT_PATH);
+        options.addOption(OPTION_BACKEND);
         return options;
     }
 
@@ -85,7 +93,21 @@ public class EMADLGeneratorCli {
     private static void runGenerator(CommandLine cliArgs) {
         String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
         String outputPath = cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt());
-        EMADLGenerator generator = new EMADLGenerator();
+        String backendString = cliArgs.getOptionValue(OPTION_BACKEND.getOpt());
+        final String DEFAULT_BACKEND = "MXNET";
+
+        if (backendString == null) {
+            Log.warn("backend not specified. backend set to default value " + DEFAULT_BACKEND);
+            backendString = DEFAULT_BACKEND;
+        }
+
+        Optional<Backend> backend = Backend.getBackendFromString(backendString);
+        if (!backend.isPresent()){
+            Log.warn("specified backend " + backendString + " not supported. backend set to default value " + DEFAULT_BACKEND);
+            backend = Backend.getBackendFromString(DEFAULT_BACKEND);
+        }
+        EMADLGenerator generator = new EMADLGenerator(backend.get());
+
         if (outputPath != null){
             generator.setGenerationTargetPath(outputPath);
         }

src/main/java/de/monticore/lang/monticar/emadl/generator/TemplateConfiguration.java

-/**
- *
- *  ******************************************************************************
- *  MontiCAR Modeling Family, www.se-rwth.de
- *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
- *  All rights reserved.
- *
- *  This project is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 3.0 of the License, or (at your option) any later version.
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
- * *******************************************************************************
- */
-package de.monticore.lang.monticar.emadl.generator;
-
-import freemarker.template.Configuration;
-import freemarker.template.TemplateExceptionHandler;
-
-public class TemplateConfiguration {
-
-    private static TemplateConfiguration instance;
-    private Configuration configuration;
-
-    private TemplateConfiguration() {
-        configuration = new Configuration(Configuration.VERSION_2_3_23);
-        configuration.setClassForTemplateLoading(TemplateConfiguration.class, "/templates/");
-        configuration.setDefaultEncoding("UTF-8");
-        configuration.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
-    }
-
-    public Configuration getConfiguration() {
-        return configuration;
-    }
-
-    public static Configuration get(){
-        if (instance == null){
-            instance = new TemplateConfiguration();
-        }
-        return instance.getConfiguration();
-    }
-}

src/main/resources/templates/CNNTrainer.ftl

-import logging
-import mxnet as mx
-<#list instances as instance>
-import CNNCreator_${instance.fullName?replace(".", "_")}
-</#list>
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    logger = logging.getLogger()
-    handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
-    logger.addHandler(handler)
-
-<#list instances as instance>
-    ${instance.fullName?replace(".", "_")} = CNNCreator_${instance.fullName?replace(".", "_")}.CNNCreator_${instance.fullName?replace(".", "_")}()
-    ${instance.fullName?replace(".", "_")}.train(
-    <#if (trainParams[instance_index])??>
-       ${trainParams[instance_index]}
-    </#if>
-    )
-
-</#list>
\ No newline at end of file

src/test/java/de/monticore/lang/monticar/emadl/AbstractSymtabTest.java

@@ -20,7 +20,7 @@
  */
 package de.monticore.lang.monticar.emadl;
 
-import de.monticore.lang.monticar.emadl.generator.AbstractSymtab;
+import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
 import de.monticore.lang.tagging._symboltable.TaggingResolver;
 import org.junit.Assert;
 
@@ -36,7 +36,7 @@ import static org.junit.Assert.assertTrue;
 public class AbstractSymtabTest {
     protected static TaggingResolver createSymTab(String... modelPath) {
 
-        return AbstractSymtab.createSymTabAndTaggingResolver(modelPath);
+        return EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelPath);
     }
 
     public static void checkFilesAreEqual(Path generationPath, Path resultsPath, List<String> fileNames) {

src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java

@@ -44,7 +44,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testCifar10Generation() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
 
@@ -65,7 +65,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testSimulatorGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -73,7 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testAddGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "Add"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "Add", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -81,7 +81,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testAlexnetGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -89,7 +89,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testResNeXtGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -97,7 +97,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testThreeInputGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().size() == 1);
     }
@@ -105,7 +105,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testMultipleOutputsGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleOutputs"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleOutputs", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().size() == 1);
     }
@@ -113,7 +113,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void tesVGGGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "VGG16"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "VGG16", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -121,7 +121,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testMultipleInstances() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "InstanceTest.MainB"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "InstanceTest.MainB", "-b", "MXNET"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }

src/test/java/de/monticore/lang/monticar/emadl/SymtabTest.java

@@ -26,7 +26,6 @@ import de.monticore.lang.monticar.emadl._parser.EMADLParser;
 import de.monticore.symboltable.Scope;
 import de.se_rwth.commons.logging.Log;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertNotNull;
@@ -48,7 +47,6 @@ public class SymtabTest extends AbstractSymtabTest {
     }
 
     @Test
-    @Ignore
     public void testAlexnet(){
         Scope symTab = createSymTab("src/test/resources/models");
         ComponentSymbol a = symTab.<ComponentSymbol>resolve("ResNet34", ComponentSymbol.KIND).orElse(null);

src/test/resources/models/InstanceTest/MainB.emadl

@@ -19,7 +19,4 @@ component MainB{
     connect net2.predictions -> outCalc2.probabilities;
     connect outCalc1.digit -> digit1;
     connect outCalc2.digit -> digit2;
-
-
-    }
 }
\ No newline at end of file

src/test/resources/models/cifar10/Cifar10Classifier.emadl

@@ -13,6 +13,4 @@ component Cifar10Classifier{
     connect image -> net.data;
     connect net.softmax -> calculateClass.inputVector;
     connect calculateClass.maxIndex -> classIndex;
-
-    }
 }
\ No newline at end of file

src/test/resources/target_code/CNNCreator_cifar10_cifar10Classifier_net.py

@@ -105,8 +105,9 @@ class CNNCreator_cifar10_cifar10Classifier_net:
             sys.exit(1)
 
 
-    def train(self, batch_size,
+    def train(self, batch_size=64,
               num_epoch=10,
+              eval_metric='acc',
               optimizer='adam',
               optimizer_params=(('learning_rate', 0.001),),
               load_checkpoint=True,
@@ -158,6 +159,7 @@ class CNNCreator_cifar10_cifar10Classifier_net:
 
         self.module.fit(
             train_data=train_iter,
+            eval_metric=eval_metric,
             eval_data=test_iter,
             optimizer=optimizer,
             optimizer_params=optimizer_params,

src/test/resources/target_code/CNNPredictor_cifar10_cifar10Classifier_net.h

@@ -29,7 +29,7 @@ public:
     }
 
     void predict(const std::vector<float> &data,
-                 std::vector<float> &softmax){
+                     std::vector<float> &softmax){
         MXPredSetInput(handle, "data", data.data(), data.size());
         //MXPredSetInput(handle, "data", data.data(), data.size());
 

src/test/resources/target_code/CNNTrainer_cifar10_Cifar10Classifier.py

@@ -5,24 +5,20 @@ import CNNCreator_cifar10_cifar10Classifier_net
 if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG)
     logger = logging.getLogger()
-    handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
+    handler = logging.FileHandler("train.log", "w", encoding=None, delay="true")
     logger.addHandler(handler)
 
     cifar10_cifar10Classifier_net = CNNCreator_cifar10_cifar10Classifier_net.CNNCreator_cifar10_cifar10Classifier_net()
     cifar10_cifar10Classifier_net.train(
-       batch_size = 64,
-num_epoch = 10,
-load_checkpoint = False,
-context = 'gpu',
-normalize = True,
-optimizer = 'adam',
-optimizer_params = {
-    'weight_decay': 1.0E-4,
-    'learning_rate': 0.01,
-    'learning_rate_decay': 0.8,
-    'step_size': 1000}
-
-
-
+        batch_size=64,
+        num_epoch=10,
+        load_checkpoint=False,
+        context='gpu',
+        normalize=True,
+        optimizer='adam',
+        optimizer_params={
+            'weight_decay': 1.0E-4,
+            'learning_rate': 0.01,
+            'learning_rate_decay': 0.8,
+            'step_size': 1000}
     )
-

src/test/resources/target_code/cifar10_cifar10Classifier.h

@@ -9,13 +9,13 @@
 using namespace arma;
 class cifar10_cifar10Classifier{
 public:
-cube image;
-double classIndex;
+icube image;
+int classIndex;
 cifar10_cifar10Classifier_net net;
 cifar10_cifar10Classifier_calculateClass calculateClass;
 void init()
 {
-image = cube(3, 32, 32);
+image = icube(3, 32, 32);
 net.init();
 calculateClass.init();
 }

src/test/resources/target_code/cifar10_cifar10Classifier_calculateClass.h

@@ -9,7 +9,7 @@ class cifar10_cifar10Classifier_calculateClass{
 const int n = 10;
 public:
 colvec inputVector;
-double maxIndex;
+int maxIndex;
 void init()
 {
 inputVector=colvec(n);
@@ -17,11 +17,11 @@ inputVector=colvec(n);
 void execute()
 {
 maxIndex = 0;
-double maxValue = inputVector(0);
+double maxValue = inputVector(0-1);
 for( auto i=2;i<=n;++i){
-if((inputVector(i) > maxValue)){
+if((inputVector(i-1) > maxValue)){
 maxIndex = i-1;
-maxValue = inputVector(i-1) ;
+maxValue = inputVector(i-1);
 }
 }
 }

src/test/resources/target_code/cifar10_cifar10Classifier_net.h

@@ -11,11 +11,11 @@ class cifar10_cifar10Classifier_net{
 const int classes = 10;
 public:
 CNNPredictor_cifar10_cifar10Classifier_net _cnn_;
-cube data;
+icube data;
 colvec softmax;
 void init()
 {
-data = cube(3, 32, 32);
+data = icube(3, 32, 32);
 softmax=colvec(classes);
 }
 void execute(){