Commit db1f942b authored by Svetlana Pavlitskaya's avatar Svetlana Pavlitskaya Committed by Evgeny Kusmenko

EMADL support

parent 5e7ffba8
......@@ -12,12 +12,12 @@ masterJobLinux:
only:
- master
masterJobWindows:
stage: windows
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
tags:
- Windows10
#masterJobWindows:
# stage: windows
# script:
# - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
# tags:
# - Windows10
BranchJobLinux:
stage: linux
......
......@@ -9,7 +9,7 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-math-middleware-generator</artifactId>
<version>0.0.7-SNAPSHOT</version>
<version>0.0.8-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
......@@ -20,6 +20,8 @@
<Embedded-montiarc-math-generator.version>0.0.25-SNAPSHOT</Embedded-montiarc-math-generator.version>
<Embedded-montiarc-math-roscpp-generator.version>0.0.4-SNAPSHOT</Embedded-montiarc-math-roscpp-generator.version>
<Embedded-montiarc-math-rosmsg-generator.version>0.0.3-SNAPSHOT</Embedded-montiarc-math-rosmsg-generator.version>
<EMADL.version>0.2.2-SNAPSHOT</EMADL.version>
<EMADL2CPP.version>0.2.2-SNAPSHOT</EMADL2CPP.version>
<!--TODO: remove with update to new emam version-->
<Tagging.version>0.0.6</Tagging.version>
<!-- .. Libraries .................................................. -->
......@@ -83,6 +85,18 @@
<version>${Embedded-montiarc-math-rosmsg-generator.version}</version>
</dependency>
<!-- EMADL Dependencies -->
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-deeplearning</artifactId>
<version>${EMADL.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>${EMADL2CPP.version}</version>
</dependency>
<!-- MontiCore Dependencies -->
<dependency>
......
......@@ -2,7 +2,10 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosToEmamTagSchema;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.monticore.lang.monticar.generator.middleware.impls.CPPGenImpl;
import de.monticore.lang.monticar.generator.middleware.impls.EMADLGeneratorImpl;
import de.monticore.lang.monticar.generator.middleware.impls.ODVGenImpl;
import de.monticore.lang.monticar.generator.middleware.impls.RosCppGenImpl;
import de.monticore.lang.monticar.generator.order.simulator.AbstractSymtab;
......@@ -59,7 +62,10 @@ public final class DistributedTargetGeneratorCli {
.valueSeparator(',')
.build();
private static final Option OPTION_EMADL_BACKEND = EMADLGeneratorCli.OPTION_BACKEND;
public static final String GENERATOR_CPP = "cpp";
public static final String GENERATOR_EMADL = "emadlcpp";
public static final String GENERATOR_ROSCPP = "roscpp";
public static final String GENERATOR_ODV = "odv";
......@@ -82,12 +88,14 @@ public final class DistributedTargetGeneratorCli {
options.addOption(OPTION_ROOT_MODEL);
options.addOption(OPTION_OUTPUT_PATH);
options.addOption(OPTION_GENERATORS);
options.addOption(OPTION_EMADL_BACKEND);
return options;
}
private static Set<String> getGeneratorNames() {
HashSet<String> res = new HashSet<>();
res.add(GENERATOR_CPP);
res.add(GENERATOR_EMADL);
res.add(GENERATOR_ROSCPP);
res.add(GENERATOR_ODV);
return res;
......@@ -114,7 +122,13 @@ public final class DistributedTargetGeneratorCli {
String outputPath = expandHomeDir(cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt()));
String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
Set<String> generators = Arrays.stream(cliArgs.getOptionValues(OPTION_GENERATORS.getOpt())).collect(Collectors.toSet());
TaggingResolver taggingResolver = AbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
TaggingResolver taggingResolver;
if (generators.contains(GENERATOR_EMADL)) {
taggingResolver = EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
}
else{
taggingResolver = AbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
}
DistributedTargetGenerator generator = new DistributedTargetGenerator();
generator.setGenerationTargetPath(outputPath);
......@@ -139,6 +153,11 @@ public final class DistributedTargetGeneratorCli {
generator.add(new CPPGenImpl(), "cpp");
}
if (generators.contains(GENERATOR_EMADL)) {
String backendString = cliArgs.getOptionValue(OPTION_EMADL_BACKEND.getOpt());
generator.add(new EMADLGeneratorImpl(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()), backendString), "cpp");
}
if (generators.contains(GENERATOR_ROSCPP)) {
generator.add(new RosCppGenImpl(), "roscpp");
RosToEmamTagSchema.registerTagTypes(taggingResolver);
......
package de.monticore.lang.monticar.generator.middleware.impls;
import de.monticore.lang.monticar.emadl.generator.Backend;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class EMADLGeneratorImpl implements GeneratorImpl {
private String generationTargetPath;
private EMADLGenerator emadlGenerator;
final String DEFAULT_BACKEND = "MXNET";
public EMADLGeneratorImpl(String modelPath, String backendString){
Optional<Backend> backend = Backend.getBackendFromString(backendString);
if (!backend.isPresent()){
Log.warn("specified backend " + backendString + " not supported. backend set to default value " + DEFAULT_BACKEND);
backend = Backend.getBackendFromString(DEFAULT_BACKEND);
}
emadlGenerator = new EMADLGenerator(backend.get());
emadlGenerator.setModelsPath(modelPath);
}
@Override
public List<File> generate(ExpandedComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException {
List<File> files = new ArrayList<>();
emadlGenerator.setGenerationTargetPath(generationTargetPath);
List<FileContent> fileContents = emadlGenerator.generateStrings(taggingResolver, componentInstanceSymbol, taggingResolver);
for (FileContent fileContent : fileContents) {
files.add(emadlGenerator.getEmamGen().generateFile(fileContent));
}
files.add(emadlGenerator.getEmamGen().generateFile((generateCMake(componentInstanceSymbol))));
return files;
}
@Override
public void setGenerationTargetPath(String path) {
this.generationTargetPath = path;
}
private FileContent generateCMake(ExpandedComponentInstanceSymbol componentInstanceSymbol) {
FileContent cmake = new FileContent();
cmake.setFileName("CMakeLists.txt");
String name = NameHelper.getNameTargetLanguage(componentInstanceSymbol.getFullName());
cmake.setFileContent(TemplateHelper.getCoordinatorCmakeListsTemplate().replace("${compName}", name));
cmake.setFileContent(cmake.getFileContent() + "target_link_libraries(" + name + " -lmxnet)");
return cmake;
}
}
......@@ -2,10 +2,8 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.LogConfig;
import de.se_rwth.commons.logging.Finding;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.commons.lang3.ArrayUtils;
import org.junit.*;
import java.nio.file.Files;
import java.nio.file.Paths;
......@@ -23,6 +21,7 @@ public class CliTest{
private static final String INVALID_ROOT_MODEL_OPTION = "--root-model=invalid.invalid.addComp";
private static final String INVALID_GENERATOR_OPTION = "--generators=invalid";
private static final String INVALID_GENERATOR_EMPTY_OPTION = "--generators=";
public static final String RESNET_MODELNAME = "tests.emadlTests.resNet34";
@BeforeClass
public static void initLog(){
......@@ -86,8 +85,6 @@ public class CliTest{
VALID_GENERATOR_CPP_OPTION,
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
String[] positiveFileNames = {
"CMakeLists.txt",
"tests_a_addComp/cpp/tests_a_addComp.h",
......@@ -101,6 +98,68 @@ public class CliTest{
}
}
@Test
public void testSingleEMADLGenerator(){
String targetDir = "target/cliTest/SingleEmadlTest/";
String[] args = {
VALID_MODELS_DIR_OPTION,
"--root-model="+RESNET_MODELNAME,
"--generators=emadlcpp",
"--backend=MXNET",
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
String[] positiveFileNames = getEMADLGeneratedFilesList(false);
for (String positiveFileName : positiveFileNames) {
assertTrue(Files.exists(Paths.get(targetDir + positiveFileName)));
}
}
@Test
public void testEMADLAndRosGenerator(){
String targetDir = "target/cliTest/EmadlRosTest/";
String[] args = {
VALID_MODELS_DIR_OPTION,
"--root-model="+RESNET_MODELNAME,
"--generators=emadlcpp,roscpp",
"--backend=MXNET",
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
String[] positiveFileNames = getEMADLGeneratedFilesList(true);
for (String positiveFileName : positiveFileNames) {
assertTrue(Files.exists(Paths.get(targetDir + positiveFileName)));
}
}
private String[] getEMADLGeneratedFilesList(boolean generateRosFiles) {
String modelName = RESNET_MODELNAME.replace('.', '_');
String[] generatedFiles = {
modelName + "/CMakeLists.txt",
modelName + "/cpp/CNNBufferFile.h",
modelName + "/cpp/CNNCreator_" + modelName + ".py",
modelName + "/cpp/CNNPredictor_" + modelName + ".h",
modelName + "/cpp/CNNTrainer_tests_emadlTests_ResNet34.py",
modelName + "/cpp/CNNTranslator.h",
modelName + "/cpp/HelperA.h",
modelName + "/cpp/CMakeLists.txt",
modelName + "/cpp/" + modelName + ".h",
modelName + "/coordinator/CMakeLists.txt",
modelName + "/coordinator/Coordinator_" + modelName + ".cpp",
modelName + "/coordinator/IAdapter_" + modelName + ".h"
};
if (generateRosFiles) {
String[] generatedRosFiles = {
modelName + "/roscpp/CMakeLists.txt",
modelName + "/roscpp/RosAdapter_" + modelName + ".h"
};
generatedFiles = ArrayUtils.addAll(generatedFiles, generatedRosFiles);
}
return generatedFiles;
}
@Test
public void testAllGenerators(){
String targetDir = "target/cliTest/AllGenerators/";
......@@ -111,7 +170,6 @@ public class CliTest{
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
String[] positiveFileNames = {
"CMakeLists.txt",
"tests_a_addComp/cpp/tests_a_addComp.h",
......
package tests.aEmadl;
conforms to de.monticore.lang.monticar.generator.roscpp.RosToEmamTagSchema;
tags Add {
tag addComp.in1 with RosConnection = {topic = (/clock, rosgraph_msgs/Clock), msgField = clock.toSec()};
tag addComp.out1 with RosConnection = {topic = (/echo, std_msgs/Float64), msgField = data};
}
package tests.aEmadl;
component AddComp{
ports in Q in1,
in Q in2,
out Q out1;
implementation Math{
out1 = in1 + in2;
}
}
\ No newline at end of file
package tests.aEmadl;
component CompA{
port in Q rosIn;
port in Q noRosIn;
port out Q rosOut;
port out Q noRosOut;
implementation Math{
rosOut = rosIn;
}
}
\ No newline at end of file
package tests.aEmadl;
conforms to de.monticore.lang.monticar.generator.roscpp.RosToEmamTagSchema;
tags Echo {
tag compA.rosIn with RosConnection = {topic = (/clock, rosgraph_msgs/Clock), msgField = clock.toSec()};
tag compA.rosOut with RosConnection = {topic = (/echo, std_msgs/Float64), msgField = data};
}
configuration ResNet34{
num_epoch:10
batch_size:64
normalize:true
load_checkpoint:false
optimizer:adam{
learning_rate:0.01
learning_rate_decay:0.8
step_size:1000
}
}
\ No newline at end of file
package tests.emadlTests;
component ResNet34{
ports in Z(0:255)^{3, 224, 224} image,
out Q(0:1)^{1000} predictions;
implementation CNN {
def conv(kernel, channels, stride=1, act=true){
Convolution(kernel=(kernel,kernel), channels=channels, stride=(stride,stride)) ->
BatchNorm() ->
Relu(?=act)
}
def resLayer(channels, stride=1){
(
conv(kernel=3, channels=channels, stride=stride) ->
conv(kernel=3, channels=channels, act=false)
|
conv(kernel=1, channels=channels, stride=stride, act=false, ? = (stride != 1))
) ->
Add() ->
Relu()
}
image ->
conv(kernel=7, channels=64, stride=2) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2)) ->
resLayer(channels=64, ->=3) ->
resLayer(channels=128, stride=2) ->
resLayer(channels=128, ->=3) ->
resLayer(channels=256, stride=2) ->
resLayer(channels=256, ->=5) ->
resLayer(channels=512, stride=2) ->
resLayer(channels=512, ->=2) ->
GlobalPooling(pool_type="avg") ->
FullyConnected(units=1000) ->
Softmax() ->
predictions
}
}
\ No newline at end of file
package tests.emadlTests;
conforms to de.monticore.lang.monticar.generator.roscpp.RosToEmamTagSchema;
tags ResNet34 {
tag resNet34.image with RosConnection = {topic=(/image, std_msgs/Int32MultiArray)};
tag resNet34.predictions with RosConnection = {topic=(/predictions, std_msgs/Float64MultiArray)};
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment