Merge branch 'adapt_different_backends' into 'master'

Adapt different backends, moved to a newer EMADL version

See merge request !5

.gitlab-ci.yml

@@ -28,20 +28,23 @@ masterJobLinux:
   image: maven:3-jdk-8
   script: 
   - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean deploy --settings settings.xml
+  - cat target/site/jacoco/index.html
+  - mvn package sonar:sonar -s settings.xml
   only:
   - master
   
-masterJobWindows:
-  stage: windows
-  script: 
-  - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
-  tags:
-  - Windows10
+#masterJobWindows:
+#  stage: windows
+#  script:
+#  - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
+#  tags:
+#  - Windows10
 
 BranchJobLinux:
   stage: linux
   image: maven:3-jdk-8
   script: 
   - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
+  - cat target/site/jacoco/index.html
   except:
   - master

README.md

+![pipeline](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/build.svg)
+![coverage](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/coverage.svg)
 # EMADL2CPP
 Generates CPP/Python code for EmbeddedMontiArcDL.
 See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/EMADL-Demo) for more information on how the generated code can be used.
\ No newline at end of file

pom.xml

@@ -8,17 +8,18 @@
 
   <groupId>de.monticore.lang.monticar</groupId>
   <artifactId>embedded-montiarc-emadl-generator</artifactId>
-  <version>0.2.1-SNAPSHOT</version>
+  <version>0.2.2-SNAPSHOT</version>
 
   <!-- == PROJECT DEPENDENCIES ============================================= -->
 
   <properties>
 
     <!-- .. SE-Libraries .................................................. -->
-    <emadl.version>0.2.1-SNAPSHOT</emadl.version>
-    <CNNTrain.version>0.2.1-SNAPSHOT</CNNTrain.version>
-    <cnnarch-generator.version>0.2.1-SNAPSHOT</cnnarch-generator.version>
-    <embedded-montiarc-math-generator>0.0.10</embedded-montiarc-math-generator>
+    <emadl.version>0.2.2-SNAPSHOT</emadl.version>
+    <CNNTrain.version>0.2.4-SNAPSHOT</CNNTrain.version>
+    <cnnarch-mxnet-generator.version>0.2.3-SNAPSHOT</cnnarch-mxnet-generator.version>
+    <cnnarch-caffe2-generator.version>0.2.2-SNAPSHOT</cnnarch-caffe2-generator.version>
+    <embedded-montiarc-math-generator>0.0.25-SNAPSHOT</embedded-montiarc-math-generator>
     
         <!-- .. Libraries .................................................. -->
         <guava.version>18.0</guava.version>
@@ -31,6 +32,7 @@
         <compiler.plugin>3.3</compiler.plugin>
         <source.plugin>2.4</source.plugin>
         <shade.plugin>2.4.3</shade.plugin>
+        <jacoco.plugin>0.8.1</jacoco.plugin>
 
     <!-- Classifiers -->
     <grammars.classifier>grammars</grammars.classifier>
@@ -68,7 +70,13 @@
         <dependency>
             <groupId>de.monticore.lang.monticar</groupId>
             <artifactId>cnnarch-mxnet-generator</artifactId>
-            <version>${cnnarch-generator.version}</version>
+            <version>${cnnarch-mxnet-generator.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>de.monticore.lang.monticar</groupId>
+            <artifactId>cnnarch-caffe2-generator</artifactId>
+            <version>${cnnarch-caffe2-generator.version}</version>
         </dependency>
 
         <dependency>
@@ -132,6 +140,27 @@
         <version>2.8.1</version>
       </plugin>
 
+        <!-- Test coverage -->
+        <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.plugin}</version>
+            <executions>
+                <execution>
+                    <id>pre-unit-test</id>
+                    <goals>
+                        <goal>prepare-agent</goal>
+                    </goals>
+                </execution>
+                <execution>
+                    <id>post-unit-test</id>
+                    <phase>test</phase>
+                    <goals>
+                        <goal>report</goal>
+                    </goals>
+                </execution>
+            </executions>
+        </plugin>
       <!-- Other Configuration -->
       <plugin>
         <artifactId>maven-compiler-plugin</artifactId>

settings.xml

@@ -57,6 +57,24 @@
   </mirrors>
 
   <profiles>
+    <profile>
+      <id>sonar</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <properties>
+        <!-- Optional URL to server. Default value is http://localhost:9000 -->
+        <sonar.host.url>
+          https://metric.se.rwth-aachen.de
+        </sonar.host.url>
+        <sonar.login>
+          jenkins
+        </sonar.login>
+        <sonar.password>
+          ${env.sonar}
+        </sonar.password>
+      </properties>
+    </profile>
 
     <profile>
       <id>se-nexus</id>

src/main/java/de/monticore/lang/monticar/emadl/generator/Backend.java

+package de.monticore.lang.monticar.emadl.generator;
+
+
+import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
+import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
+import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
+
+import java.util.Optional;
+
+public enum Backend {
+    MXNET{
+        @Override
+        public CNNArchGenerator getGenerator() {
+            return new CNNArch2MxNet();
+        }
+    },
+    CAFFE2{
+        @Override
+        public CNNArchGenerator getGenerator() {
+            return new CNNArch2Caffe2();
+        }
+    };
+
+    public abstract CNNArchGenerator getGenerator();
+
+    public static Optional<Backend> getBackendFromString(String backend){
+        switch (backend){
+            case "MXNET":
+                return Optional.of(MXNET);
+
+            case "CAFFE2":
+                return Optional.of(CAFFE2);
+
+            default:
+                return Optional.empty();
+        }
+    }
+}

src/main/java/de/monticore/lang/monticar/emadl/generator/AbstractSymtab.java → src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLAbstractSymtab.java

@@ -44,7 +44,10 @@ import de.monticore.symboltable.Scope;
 import java.nio.file.Paths;
 import java.util.Arrays;
 
-public class AbstractSymtab {
+public class EMADLAbstractSymtab {
+    public EMADLAbstractSymtab() {
+    }
+
     public static TaggingResolver createSymTabAndTaggingResolver(String... modelPath) {
         Scope scope = createSymTab(modelPath);
         TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGenerator.java

@@ -22,14 +22,18 @@ package de.monticore.lang.monticar.emadl.generator;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
 import de.monticore.io.paths.ModelPath;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ComponentSymbol;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
-import de.monticore.lang.math.math._symboltable.MathStatementsSymbol;
+import de.monticore.lang.math._symboltable.MathStatementsSymbol;
+import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
 import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
-import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
+import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
+import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainCompilationUnitSymbol;
 import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainLanguage;
-import de.monticore.lang.monticar.cnntrain.generator.CNNTrainGenerator;
+import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
 import de.monticore.lang.monticar.emadl._cocos.EMADLCocos;
 import de.monticore.lang.monticar.generator.FileContent;
 import de.monticore.lang.monticar.generator.cpp.ArmadilloHelper;
@@ -42,11 +46,10 @@ import de.monticore.symboltable.GlobalScope;
 import de.monticore.symboltable.Scope;
 import de.se_rwth.commons.Splitters;
 import de.se_rwth.commons.logging.Log;
-import freemarker.template.Template;
 import freemarker.template.TemplateException;
 
-import java.io.IOException;
-import java.io.StringWriter;
+import java.io.*;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -55,19 +58,19 @@ import java.util.*;
 
 public class EMADLGenerator {
 
-    public static final String CNN_HELPER = "CNNTranslator";
-    public static final String CNN_TRAINER = "CNNTrainer";
-
     private GeneratorCPP emamGen;
+    private CNNArchGenerator cnnArchGenerator;
+
+    private String modelsPath;
 
-    public EMADLGenerator() {
+
+    public EMADLGenerator(Backend backend) {
         emamGen = new GeneratorCPP();
         emamGen.useArmadilloBackend();
         emamGen.setGenerationTargetPath("./target/generated-sources-emadl/");
+        cnnArchGenerator = backend.getGenerator();
     }
 
-    private String modelsPath;
-
     public String getModelsPath() {
         return modelsPath;
     }
@@ -100,7 +103,7 @@ public class EMADLGenerator {
 
     public void generate(String modelPath, String qualifiedName) throws IOException, TemplateException {
         setModelsPath( modelPath );
-        TaggingResolver symtab = AbstractSymtab.createSymTabAndTaggingResolver(getModelsPath());
+        TaggingResolver symtab = EMADLAbstractSymtab.createSymTabAndTaggingResolver(getModelsPath());
         ComponentSymbol component = symtab.<ComponentSymbol>resolve(qualifiedName, ComponentSymbol.KIND).orElse(null);
 
         List<String> splitName = Splitters.DOT.splitToList(qualifiedName);
@@ -131,7 +134,7 @@ public class EMADLGenerator {
 
         generateComponent(fileContents, allInstances, taggingResolver, componentInstanceSymbol, symtab);
 
-        fileContents.add(generateCNNTrainer(allInstances, componentInstanceSymbol.getComponentType().getFullName().replaceAll("\\.", "_")));
+        fileContents.addAll(generateCNNTrainer(allInstances, componentInstanceSymbol.getComponentType().getFullName().replaceAll("\\.", "_")));
         fileContents.add(ArmadilloHelper.getArmadilloHelperFileContent());
         TypesGeneratorCPP tg = new TypesGeneratorCPP();
         fileContents.addAll(tg.generateTypes(TypeConverter.getTypeSymbols()));
@@ -185,7 +188,6 @@ public class EMADLGenerator {
     }
 
     public void generateCNN(List<FileContent> fileContents, TaggingResolver taggingResolver, ExpandedComponentInstanceSymbol instance, ArchitectureSymbol architecture){
-        CNNArchGenerator cnnArchGenerator = new CNNArchGenerator();
         Map<String,String> contentMap = cnnArchGenerator.generateStrings(architecture);
         String fullName = instance.getFullName().replaceAll("\\.", "_");
 
@@ -206,7 +208,7 @@ public class EMADLGenerator {
             fileContents.add(new FileContent(contentMap.get(fileName), fileName));
         }
         fileContents.add(componentFileContent);
-        fileContents.add(new FileContent(processTemplate(new HashMap<>(), CNN_HELPER), CNN_HELPER + ".h"));
+        fileContents.add(new FileContent(readResource("CNNTranslator.h", Charsets.UTF_8), "CNNTranslator.h"));
     }
 
     protected String transformComponent(String component, String predictorClassName, String executeMethod){
@@ -215,7 +217,7 @@ public class EMADLGenerator {
         //insert includes
         component = component.replaceFirst("using namespace",
                 "#include \"" + predictorClassName + ".h" + "\"\n" +
-                        "#include \"" + CNN_HELPER + ".h" + "\"\n" +
+                        "#include \"CNNTranslator.h\"\n" +
                         "using namespace");
 
         //insert network attribute
@@ -252,34 +254,28 @@ public class EMADLGenerator {
         }
     }
 
-    public FileContent generateCNNTrainer(Set<ExpandedComponentInstanceSymbol> allInstances, String mainComponentName){
-        List<ExpandedComponentInstanceSymbol> cnnInstances = new ArrayList<>();
-        List<String> trainParams = new ArrayList<>();
-        Set<String> componentNames = new HashSet<>();
-        for (ExpandedComponentInstanceSymbol componentInstance : allInstances){
+    public List<FileContent> generateCNNTrainer(Set<ExpandedComponentInstanceSymbol> allInstances, String mainComponentName) {
+        List<String> cnnInstanceNames = new ArrayList<>();
+        List<ConfigurationSymbol> configurations = new ArrayList<>();
+        for (ExpandedComponentInstanceSymbol componentInstance : allInstances) {
             ComponentSymbol component = componentInstance.getComponentType().getReferencedSymbol();
             Optional<ArchitectureSymbol> architecture = component.getSpannedScope().resolve("", ArchitectureSymbol.KIND);
 
-            if (architecture.isPresent()){
-
-                String fileContent = getTrainingParamsForComponent(mainComponentName, component, componentInstance);
-                if (!fileContent.isEmpty()) {
-                    trainParams.add(fileContent);
-                }
-
-                cnnInstances.add(componentInstance);
-                componentNames.add(component.getFullName());
-
+            if (architecture.isPresent()) {
+                ConfigurationSymbol configuration = getTrainingConfiguration(mainComponentName, component, componentInstance);
+                configurations.add(configuration);
+                cnnInstanceNames.add(componentInstance.getFullName().replaceAll("\\.", "_"));
             }
         }
-        Map<String, Object> ftlContext = new HashMap<>();
-        ftlContext.put("instances", cnnInstances);
-        ftlContext.put("componentNames", componentNames);
-        ftlContext.put("trainParams", trainParams);
-        return new FileContent(processTemplate(ftlContext, CNN_TRAINER), CNN_TRAINER + "_" + mainComponentName + ".py");
+        List<FileContent> fileContents = new ArrayList<>();
+        Map<String, String> fileContentMap =  cnnArchGenerator.generateTrainer(configurations, cnnInstanceNames, mainComponentName);
+        for (String fileName : fileContentMap.keySet()){
+            fileContents.add(new FileContent(fileContentMap.get(fileName), fileName));
+        }
+        return fileContents;
     }
 
-    private String getTrainingParamsForComponent(String mainComponentName, ComponentSymbol component, ExpandedComponentInstanceSymbol instance) {
+    public ConfigurationSymbol getTrainingConfiguration(String mainComponentName, ComponentSymbol component, ExpandedComponentInstanceSymbol instance) {
         String configFilename;
         String mainComponentConfigFilename = mainComponentName.replaceAll("\\.", "/");
         String componentConfigFilename = component.getFullName().replaceAll("\\.", "/");
@@ -300,7 +296,7 @@ public class EMADLGenerator {
                     + getModelsPath() + componentConfigFilename + ".cnnt', '"
                     + getModelsPath() + mainComponentConfigFilename + ".cnnt'." +
                     " These files denote respectively the configuration for the single instance, the component or the whole system.");
-            return "";
+            return null;
         }
 
         //should be removed when CNNTrain supports packages
@@ -309,29 +305,32 @@ public class EMADLGenerator {
         Path modelPath = Paths.get(getModelsPath() + Joiner.on("/").join(names.subList(0,names.size()-1)));
         //
 
-        CNNTrainGenerator cnnTrainGenerator =  new CNNTrainGenerator();
+        //CNNTrainGenerator cnnTrainGenerator =  new CNNTrainGenerator(); //No need of cnnTrainGenerator since cnnArchGenerator can also generateTrainer()
         final ModelPath mp = new ModelPath(modelPath);
         GlobalScope trainScope = new GlobalScope(mp, new CNNTrainLanguage());
-        Map.Entry<String, String> fileContents = cnnTrainGenerator.generateFileContent( trainScope, configFilename );
-        return fileContents.getValue();
+        Optional<CNNTrainCompilationUnitSymbol> compilationUnit = trainScope.resolve(configFilename, CNNTrainCompilationUnitSymbol.KIND);
+        if (!compilationUnit.isPresent()){
+            Log.error("CNNTrainCompilationUnitSymbol is empty. Could not resolve configuration " + configFilename);
+            System.exit(1);
+        }
+        CNNTrainCocos.checkAll(compilationUnit.get());
+        ConfigurationSymbol configuration = compilationUnit.get().getConfiguration();
+
+        return configuration;
     }
 
-    protected String processTemplate(Map<String, Object> ftlContext, String templateNameWithoutEnding){
-        StringWriter writer = new StringWriter();
-        String templateName = templateNameWithoutEnding + ".ftl";
-        try{
-            Template template = TemplateConfiguration.get().getTemplate(templateName);
-            template.process(ftlContext, writer);
-        }
-        catch (IOException e) {
-            Log.error("Freemarker could not find template " + templateName + " :\n" + e.getMessage());
+    public String readResource(final String fileName, Charset charset) {
+        try {
+            return Resources.toString(Resources.getResource(fileName), charset);
+
+        } catch (IllegalArgumentException e) {
+            System.err.println("Resource file " + fileName + " not found");
             System.exit(1);
-        }
-        catch (TemplateException e){
-            Log.error("An exception occured in template " + templateName + " :\n" + e.getMessage());
+            return null;
+        } catch (IOException e) {
+            System.err.println("IO Error occurred");
             System.exit(1);
+            return null;
         }
-        return writer.toString();
     }
-
 }

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGeneratorCli.java

@@ -26,6 +26,7 @@ import freemarker.template.TemplateException;
 import org.apache.commons.cli.*;
 
 import java.io.IOException;
+import java.util.Optional;
 
 public class EMADLGeneratorCli {
 
@@ -49,6 +50,12 @@ public class EMADLGeneratorCli {
             .hasArg(true)
             .required(false)
             .build();
+    public static final Option OPTION_BACKEND = Option.builder("b")
+            .longOpt("backend")
+            .desc("deep-learning-framework backend. Options: MXNET, CAFFE2")
+            .hasArg(true)
+            .required(false)
+            .build();
 
     private EMADLGeneratorCli() {
     }
@@ -67,6 +74,7 @@ public class EMADLGeneratorCli {
         options.addOption(OPTION_MODELS_PATH);
         options.addOption(OPTION_ROOT_MODEL);
         options.addOption(OPTION_OUTPUT_PATH);
+        options.addOption(OPTION_BACKEND);
         return options;
     }
 
@@ -85,7 +93,21 @@ public class EMADLGeneratorCli {
     private static void runGenerator(CommandLine cliArgs) {
         String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
         String outputPath = cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt());
-        EMADLGenerator generator = new EMADLGenerator();
+        String backendString = cliArgs.getOptionValue(OPTION_BACKEND.getOpt());
+        final String DEFAULT_BACKEND = "MXNET";
+
+        if (backendString == null) {
+            Log.warn("backend not specified. backend set to default value " + DEFAULT_BACKEND);
+            backendString = DEFAULT_BACKEND;
+        }
+
+        Optional<Backend> backend = Backend.getBackendFromString(backendString);
+        if (!backend.isPresent()){
+            Log.warn("specified backend " + backendString + " not supported. backend set to default value " + DEFAULT_BACKEND);
+            backend = Backend.getBackendFromString(DEFAULT_BACKEND);
+        }
+        EMADLGenerator generator = new EMADLGenerator(backend.get());
+
         if (outputPath != null){