EMADL support

.gitlab-ci.yml

@@ -12,12 +12,12 @@ masterJobLinux:
   only:
   - master
 
-masterJobWindows:
-  stage: windows
-  script:
-  - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
-  tags:
-  - Windows10
+#masterJobWindows:
+#  stage: windows
+#  script:
+#  - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml
+#  tags:
+#  - Windows10
 
 BranchJobLinux:
   stage: linux

pom.xml

@@ -9,7 +9,7 @@
 
     <groupId>de.monticore.lang.monticar</groupId>
     <artifactId>embedded-montiarc-math-middleware-generator</artifactId>
-    <version>0.0.7-SNAPSHOT</version>
+    <version>0.0.8-SNAPSHOT</version>
 
     <!-- == PROJECT DEPENDENCIES ============================================= -->
 
@@ -20,6 +20,8 @@
         <Embedded-montiarc-math-generator.version>0.0.25-SNAPSHOT</Embedded-montiarc-math-generator.version>
         <Embedded-montiarc-math-roscpp-generator.version>0.0.4-SNAPSHOT</Embedded-montiarc-math-roscpp-generator.version>
         <Embedded-montiarc-math-rosmsg-generator.version>0.0.3-SNAPSHOT</Embedded-montiarc-math-rosmsg-generator.version>
+        <EMADL.version>0.2.2-SNAPSHOT</EMADL.version>
+        <EMADL2CPP.version>0.2.2-SNAPSHOT</EMADL2CPP.version>
         <!--TODO: remove with update to new emam version-->
         <Tagging.version>0.0.6</Tagging.version>
         <!-- .. Libraries .................................................. -->
@@ -83,6 +85,18 @@
             <version>${Embedded-montiarc-math-rosmsg-generator.version}</version>
         </dependency>
 
+        <!-- EMADL Dependencies -->
+        <dependency>
+            <groupId>de.monticore.lang.monticar</groupId>
+            <artifactId>embedded-montiarc-deeplearning</artifactId>
+            <version>${EMADL.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>de.monticore.lang.monticar</groupId>
+            <artifactId>embedded-montiarc-emadl-generator</artifactId>
+            <version>${EMADL2CPP.version}</version>
+        </dependency>
+
         <!-- MontiCore Dependencies -->
 
         <dependency>

src/main/java/de/monticore/lang/monticar/generator/middleware/DistributedTargetGeneratorCli.java

@@ -2,7 +2,10 @@ package de.monticore.lang.monticar.generator.middleware;
 
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
 import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosToEmamTagSchema;
+import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
+import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
 import de.monticore.lang.monticar.generator.middleware.impls.CPPGenImpl;
+import de.monticore.lang.monticar.generator.middleware.impls.EMADLGeneratorImpl;
 import de.monticore.lang.monticar.generator.middleware.impls.ODVGenImpl;
 import de.monticore.lang.monticar.generator.middleware.impls.RosCppGenImpl;
 import de.monticore.lang.monticar.generator.order.simulator.AbstractSymtab;
@@ -59,7 +62,10 @@ public final class DistributedTargetGeneratorCli {
             .valueSeparator(',')
             .build();
 
+    private static final Option OPTION_EMADL_BACKEND = EMADLGeneratorCli.OPTION_BACKEND;
+
     public static final String GENERATOR_CPP = "cpp";
+    public static final String GENERATOR_EMADL = "emadlcpp";
     public static final String GENERATOR_ROSCPP = "roscpp";
     public static final String GENERATOR_ODV = "odv";
 
@@ -82,12 +88,14 @@ public final class DistributedTargetGeneratorCli {
         options.addOption(OPTION_ROOT_MODEL);
         options.addOption(OPTION_OUTPUT_PATH);
         options.addOption(OPTION_GENERATORS);
+        options.addOption(OPTION_EMADL_BACKEND);
         return options;
     }
 
     private static Set<String> getGeneratorNames() {
         HashSet<String> res = new HashSet<>();
         res.add(GENERATOR_CPP);
+        res.add(GENERATOR_EMADL);
         res.add(GENERATOR_ROSCPP);
         res.add(GENERATOR_ODV);
         return res;
@@ -114,7 +122,13 @@ public final class DistributedTargetGeneratorCli {
         String outputPath = expandHomeDir(cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt()));
         String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
         Set<String> generators = Arrays.stream(cliArgs.getOptionValues(OPTION_GENERATORS.getOpt())).collect(Collectors.toSet());
-        TaggingResolver taggingResolver = AbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
+        TaggingResolver taggingResolver;
+        if (generators.contains(GENERATOR_EMADL)) {
+            taggingResolver = EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
+        }
+        else{
+            taggingResolver = AbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
+        }
 
         DistributedTargetGenerator generator = new DistributedTargetGenerator();
         generator.setGenerationTargetPath(outputPath);
@@ -139,6 +153,11 @@ public final class DistributedTargetGeneratorCli {
             generator.add(new CPPGenImpl(), "cpp");
         }
 
+        if (generators.contains(GENERATOR_EMADL)) {
+            String backendString = cliArgs.getOptionValue(OPTION_EMADL_BACKEND.getOpt());
+            generator.add(new EMADLGeneratorImpl(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()), backendString), "cpp");
+        }
+
         if (generators.contains(GENERATOR_ROSCPP)) {
             generator.add(new RosCppGenImpl(), "roscpp");
             RosToEmamTagSchema.registerTagTypes(taggingResolver);

src/main/java/de/monticore/lang/monticar/generator/middleware/impls/EMADLGeneratorImpl.java

+package de.monticore.lang.monticar.generator.middleware.impls;
+
+import de.monticore.lang.monticar.emadl.generator.Backend;
+import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
+import de.monticore.lang.monticar.generator.FileContent;
+import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
+import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+import de.se_rwth.commons.logging.Log;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public class EMADLGeneratorImpl implements GeneratorImpl {
+    private String generationTargetPath;
+    private EMADLGenerator emadlGenerator;
+    final String DEFAULT_BACKEND = "MXNET";
+
+
+    public EMADLGeneratorImpl(String modelPath, String backendString){
+        Optional<Backend> backend = Backend.getBackendFromString(backendString);
+        if (!backend.isPresent()){
+            Log.warn("specified backend " + backendString + " not supported. backend set to default value " + DEFAULT_BACKEND);
+            backend = Backend.getBackendFromString(DEFAULT_BACKEND);
+        }
+        emadlGenerator = new EMADLGenerator(backend.get());
+        emadlGenerator.setModelsPath(modelPath);
+    }
+
+
+    @Override
+    public List<File> generate(ExpandedComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException {
+        List<File> files = new ArrayList<>();
+
+        emadlGenerator.setGenerationTargetPath(generationTargetPath);
+        List<FileContent> fileContents = emadlGenerator.generateStrings(taggingResolver, componentInstanceSymbol, taggingResolver);
+
+        for (FileContent fileContent : fileContents) {
+            files.add(emadlGenerator.getEmamGen().generateFile(fileContent));
+        }
+
+        files.add(emadlGenerator.getEmamGen().generateFile((generateCMake(componentInstanceSymbol))));
+        return files;
+    }
+
+    @Override
+    public void setGenerationTargetPath(String path) {
+        this.generationTargetPath = path;
+    }
+
+    private FileContent generateCMake(ExpandedComponentInstanceSymbol componentInstanceSymbol) {
+        FileContent cmake = new FileContent();
+        cmake.setFileName("CMakeLists.txt");
+        String name = NameHelper.getNameTargetLanguage(componentInstanceSymbol.getFullName());
+        cmake.setFileContent(TemplateHelper.getCoordinatorCmakeListsTemplate().replace("${compName}", name));
+        cmake.setFileContent(cmake.getFileContent() + "target_link_libraries(" + name + " -lmxnet)");
+        return cmake;
+    }
+}
+

src/test/java/de/monticore/lang/monticar/generator/middleware/CliTest.java

@@ -2,10 +2,8 @@ package de.monticore.lang.monticar.generator.middleware;
 
 import de.monticore.lang.embeddedmontiarc.LogConfig;
 import de.se_rwth.commons.logging.Finding;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.*;
 
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -23,6 +21,7 @@ public class CliTest{
     private static final String INVALID_ROOT_MODEL_OPTION = "--root-model=invalid.invalid.addComp";
     private static final String INVALID_GENERATOR_OPTION = "--generators=invalid";
     private static final String INVALID_GENERATOR_EMPTY_OPTION = "--generators=";
+    public static final String RESNET_MODELNAME = "tests.emadlTests.resNet34";
 
     @BeforeClass
     public static void initLog(){
@@ -86,8 +85,6 @@ public class CliTest{
                 VALID_GENERATOR_CPP_OPTION,
                 "--output-dir=" + targetDir};
         DistributedTargetGeneratorCli.main(args);
-
-
         String[] positiveFileNames = {
                 "CMakeLists.txt",
                 "tests_a_addComp/cpp/tests_a_addComp.h",
@@ -101,6 +98,68 @@ public class CliTest{
         }
     }
 
+    @Test
+    public void testSingleEMADLGenerator(){
+        String targetDir = "target/cliTest/SingleEmadlTest/";
+        String[] args = {
+                VALID_MODELS_DIR_OPTION,
+                "--root-model="+RESNET_MODELNAME,
+                "--generators=emadlcpp",
+                "--backend=MXNET",
+                "--output-dir=" + targetDir};
+        DistributedTargetGeneratorCli.main(args);
+        String[] positiveFileNames = getEMADLGeneratedFilesList(false);
+
+        for (String positiveFileName : positiveFileNames) {
+            assertTrue(Files.exists(Paths.get(targetDir + positiveFileName)));
+        }
+    }
+
+    @Test
+    public void testEMADLAndRosGenerator(){
+        String targetDir = "target/cliTest/EmadlRosTest/";
+        String[] args = {
+                VALID_MODELS_DIR_OPTION,
+                "--root-model="+RESNET_MODELNAME,
+                "--generators=emadlcpp,roscpp",
+                "--backend=MXNET",
+                "--output-dir=" + targetDir};
+        DistributedTargetGeneratorCli.main(args);
+        String[] positiveFileNames = getEMADLGeneratedFilesList(true);
+
+        for (String positiveFileName : positiveFileNames) {
+            assertTrue(Files.exists(Paths.get(targetDir + positiveFileName)));
+        }
+    }
+
+    private String[] getEMADLGeneratedFilesList(boolean generateRosFiles) {
+        String modelName = RESNET_MODELNAME.replace('.', '_');
+        String[] generatedFiles = {
+                modelName + "/CMakeLists.txt",
+
+                modelName + "/cpp/CNNBufferFile.h",
+                modelName + "/cpp/CNNCreator_" + modelName + ".py",
+                modelName + "/cpp/CNNPredictor_" + modelName + ".h",
+                modelName + "/cpp/CNNTrainer_tests_emadlTests_ResNet34.py",
+                modelName + "/cpp/CNNTranslator.h",
+                modelName + "/cpp/HelperA.h",
+                modelName + "/cpp/CMakeLists.txt",
+                modelName + "/cpp/" + modelName + ".h",
+
+                modelName + "/coordinator/CMakeLists.txt",
+                modelName + "/coordinator/Coordinator_" + modelName + ".cpp",
+                modelName + "/coordinator/IAdapter_" + modelName + ".h"
+        };
+        if (generateRosFiles) {
+            String[] generatedRosFiles = {
+                    modelName + "/roscpp/CMakeLists.txt",
+                    modelName + "/roscpp/RosAdapter_" + modelName + ".h"
+            };
+            generatedFiles = ArrayUtils.addAll(generatedFiles, generatedRosFiles);
+        }
+        return generatedFiles;
+    }
+
     @Test
     public void testAllGenerators(){
         String targetDir = "target/cliTest/AllGenerators/";
@@ -111,7 +170,6 @@ public class CliTest{
                 "--output-dir=" + targetDir};
         DistributedTargetGeneratorCli.main(args);
 
-
         String[] positiveFileNames = {
                 "CMakeLists.txt",
                 "tests_a_addComp/cpp/tests_a_addComp.h",

src/test/resources/tests/aEmadl/Add.tag

+package tests.aEmadl;
+conforms to de.monticore.lang.monticar.generator.roscpp.RosToEmamTagSchema;
+
+tags Add {
+tag addComp.in1 with RosConnection = {topic = (/clock, rosgraph_msgs/Clock), msgField = clock.toSec()};
+tag addComp.out1 with RosConnection = {topic = (/echo, std_msgs/Float64), msgField = data};
+}

src/test/resources/tests/aEmadl/AddComp.emadl

+package tests.aEmadl;
+
+component AddComp{
+    ports in Q in1,
+          in Q in2,
+          out Q out1;
+
+    implementation Math{
+        out1 = in1 + in2;
+    }
+
+}
\ No newline at end of file

src/test/resources/tests/aEmadl/CompA.emadl

+package tests.aEmadl;
+
+component CompA{
+    port in Q rosIn;
+    port in Q noRosIn;
+    port out Q rosOut;
+    port out Q noRosOut;
+
+    implementation Math{
+        rosOut = rosIn;
+    }
+}
\ No newline at end of file

src/test/resources/tests/aEmadl/Echo.tag

+package tests.aEmadl;
+conforms to de.monticore.lang.monticar.generator.roscpp.RosToEmamTagSchema;
+
+tags Echo {
+tag compA.rosIn with RosConnection = {topic = (/clock, rosgraph_msgs/Clock), msgField = clock.toSec()};
+tag compA.rosOut with RosConnection = {topic = (/echo, std_msgs/Float64), msgField = data};
+}

src/test/resources/tests/emadlTests/ResNet34.cnnt

+configuration ResNet34{
+    num_epoch:10
+    batch_size:64
+    normalize:true
+    load_checkpoint:false
+    optimizer:adam{
+        learning_rate:0.01
+        learning_rate_decay:0.8
+        step_size:1000
+    }
+}
\ No newline at end of file

src/test/resources/tests/emadlTests/ResNet34.emadl

+package tests.emadlTests;
+
+component ResNet34{
+    ports in Z(0:255)^{3, 224, 224} image,
+         out Q(0:1)^{1000} predictions;
+
+    implementation CNN {
+        def conv(kernel, channels, stride=1, act=true){
+            Convolution(kernel=(kernel,kernel), channels=channels, stride=(stride,stride)) ->
+            BatchNorm() ->
+            Relu(?=act)
+        }
+        def resLayer(channels, stride=1){
+            (
+                conv(kernel=3, channels=channels, stride=stride) ->
+                conv(kernel=3, channels=channels, act=false)
+            |
+                conv(kernel=1, channels=channels, stride=stride, act=false, ? = (stride != 1))
+            ) ->
+            Add() ->
+            Relu()
+        }
+
+        image ->
+        conv(kernel=7, channels=64, stride=2) ->
+        Pooling(pool_type="max", kernel=(3,3), stride=(2,2)) ->
+        resLayer(channels=64, ->=3) ->
+        resLayer(channels=128, stride=2) ->
+        resLayer(channels=128, ->=3) ->
+        resLayer(channels=256, stride=2) ->
+        resLayer(channels=256, ->=5) ->
+        resLayer(channels=512, stride=2) ->
+        resLayer(channels=512, ->=2) ->
+        GlobalPooling(pool_type="avg") ->
+        FullyConnected(units=1000) ->
+        Softmax() ->
+        predictions
+    }
+}
\ No newline at end of file

src/test/resources/tests/emadlTests/ResNet34.tag

+package tests.emadlTests;
+conforms to de.monticore.lang.monticar.generator.roscpp.RosToEmamTagSchema;
+
+tags ResNet34 {
+    tag resNet34.image with RosConnection = {topic=(/image, std_msgs/Int32MultiArray)};
+    tag resNet34.predictions with RosConnection = {topic=(/predictions, std_msgs/Float64MultiArray)};
+}