Commit 4e6e6673 authored by Svetlana Pavlitskaya's avatar Svetlana Pavlitskaya Committed by Evgeny Kusmenko
Browse files

Adapt different backends, moved to a newer EMADL version

parent 9c3425c2
......@@ -28,20 +28,23 @@ masterJobLinux:
image: maven:3-jdk-8
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean deploy --settings settings.xml
- cat target/site/jacoco/index.html
- mvn package sonar:sonar -s settings.xml
only:
- master
masterJobWindows:
stage: windows
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
tags:
- Windows10
#masterJobWindows:
# stage: windows
# script:
# - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
# tags:
# - Windows10
BranchJobLinux:
stage: linux
image: maven:3-jdk-8
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
- cat target/site/jacoco/index.html
except:
- master
![pipeline](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/build.svg)
![coverage](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/coverage.svg)
# EMADL2CPP
Generates CPP/Python code for EmbeddedMontiArcDL.
See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/EMADL-Demo) for more information on how the generated code can be used.
\ No newline at end of file
......@@ -8,17 +8,18 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.2.1-SNAPSHOT</version>
<version>0.2.2-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<emadl.version>0.2.1-SNAPSHOT</emadl.version>
<CNNTrain.version>0.2.1-SNAPSHOT</CNNTrain.version>
<cnnarch-generator.version>0.2.1-SNAPSHOT</cnnarch-generator.version>
<embedded-montiarc-math-generator>0.0.10</embedded-montiarc-math-generator>
<emadl.version>0.2.2-SNAPSHOT</emadl.version>
<CNNTrain.version>0.2.4-SNAPSHOT</CNNTrain.version>
<cnnarch-mxnet-generator.version>0.2.3-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.2-SNAPSHOT</cnnarch-caffe2-generator.version>
<embedded-montiarc-math-generator>0.0.25-SNAPSHOT</embedded-montiarc-math-generator>
<!-- .. Libraries .................................................. -->
<guava.version>18.0</guava.version>
......@@ -31,6 +32,7 @@
<compiler.plugin>3.3</compiler.plugin>
<source.plugin>2.4</source.plugin>
<shade.plugin>2.4.3</shade.plugin>
<jacoco.plugin>0.8.1</jacoco.plugin>
<!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier>
......@@ -68,7 +70,13 @@
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId>
<version>${cnnarch-generator.version}</version>
<version>${cnnarch-mxnet-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-caffe2-generator</artifactId>
<version>${cnnarch-caffe2-generator.version}</version>
</dependency>
<dependency>
......@@ -132,6 +140,27 @@
<version>2.8.1</version>
</plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
......
......@@ -57,6 +57,24 @@
</mirrors>
<profiles>
<profile>
<id>sonar</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<!-- Optional URL to server. Default value is http://localhost:9000 -->
<sonar.host.url>
https://metric.se.rwth-aachen.de
</sonar.host.url>
<sonar.login>
jenkins
</sonar.login>
<sonar.password>
${env.sonar}
</sonar.password>
</properties>
</profile>
<profile>
<id>se-nexus</id>
......
package de.monticore.lang.monticar.emadl.generator;
import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
import java.util.Optional;
public enum Backend {
MXNET{
@Override
public CNNArchGenerator getGenerator() {
return new CNNArch2MxNet();
}
},
CAFFE2{
@Override
public CNNArchGenerator getGenerator() {
return new CNNArch2Caffe2();
}
};
public abstract CNNArchGenerator getGenerator();
public static Optional<Backend> getBackendFromString(String backend){
switch (backend){
case "MXNET":
return Optional.of(MXNET);
case "CAFFE2":
return Optional.of(CAFFE2);
default:
return Optional.empty();
}
}
}
......@@ -44,7 +44,10 @@ import de.monticore.symboltable.Scope;
import java.nio.file.Paths;
import java.util.Arrays;
public class AbstractSymtab {
public class EMADLAbstractSymtab {
public EMADLAbstractSymtab() {
}
public static TaggingResolver createSymTabAndTaggingResolver(String... modelPath) {
Scope scope = createSymTab(modelPath);
TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));
......
......@@ -22,14 +22,18 @@ package de.monticore.lang.monticar.emadl.generator;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Charsets;
import com.google.common.io.Resources;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ComponentSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
import de.monticore.lang.math.math._symboltable.MathStatementsSymbol;
import de.monticore.lang.math._symboltable.MathStatementsSymbol;
import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainCompilationUnitSymbol;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainLanguage;
import de.monticore.lang.monticar.cnntrain.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
import de.monticore.lang.monticar.emadl._cocos.EMADLCocos;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cpp.ArmadilloHelper;
......@@ -42,11 +46,10 @@ import de.monticore.symboltable.GlobalScope;
import de.monticore.symboltable.Scope;
import de.se_rwth.commons.Splitters;
import de.se_rwth.commons.logging.Log;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import java.io.IOException;
import java.io.StringWriter;
import java.io.*;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
......@@ -55,19 +58,19 @@ import java.util.*;
public class EMADLGenerator {
public static final String CNN_HELPER = "CNNTranslator";
public static final String CNN_TRAINER = "CNNTrainer";
private GeneratorCPP emamGen;
private CNNArchGenerator cnnArchGenerator;
private String modelsPath;
public EMADLGenerator() {
public EMADLGenerator(Backend backend) {
emamGen = new GeneratorCPP();
emamGen.useArmadilloBackend();
emamGen.setGenerationTargetPath("./target/generated-sources-emadl/");
cnnArchGenerator = backend.getGenerator();
}
private String modelsPath;
public String getModelsPath() {
return modelsPath;
}
......@@ -100,7 +103,7 @@ public class EMADLGenerator {
public void generate(String modelPath, String qualifiedName) throws IOException, TemplateException {
setModelsPath( modelPath );
TaggingResolver symtab = AbstractSymtab.createSymTabAndTaggingResolver(getModelsPath());
TaggingResolver symtab = EMADLAbstractSymtab.createSymTabAndTaggingResolver(getModelsPath());
ComponentSymbol component = symtab.<ComponentSymbol>resolve(qualifiedName, ComponentSymbol.KIND).orElse(null);
List<String> splitName = Splitters.DOT.splitToList(qualifiedName);
......@@ -131,7 +134,7 @@ public class EMADLGenerator {
generateComponent(fileContents, allInstances, taggingResolver, componentInstanceSymbol, symtab);
fileContents.add(generateCNNTrainer(allInstances, componentInstanceSymbol.getComponentType().getFullName().replaceAll("\\.", "_")));
fileContents.addAll(generateCNNTrainer(allInstances, componentInstanceSymbol.getComponentType().getFullName().replaceAll("\\.", "_")));
fileContents.add(ArmadilloHelper.getArmadilloHelperFileContent());
TypesGeneratorCPP tg = new TypesGeneratorCPP();
fileContents.addAll(tg.generateTypes(TypeConverter.getTypeSymbols()));
......@@ -185,7 +188,6 @@ public class EMADLGenerator {
}
public void generateCNN(List<FileContent> fileContents, TaggingResolver taggingResolver, ExpandedComponentInstanceSymbol instance, ArchitectureSymbol architecture){
CNNArchGenerator cnnArchGenerator = new CNNArchGenerator();
Map<String,String> contentMap = cnnArchGenerator.generateStrings(architecture);
String fullName = instance.getFullName().replaceAll("\\.", "_");
......@@ -206,7 +208,7 @@ public class EMADLGenerator {
fileContents.add(new FileContent(contentMap.get(fileName), fileName));
}
fileContents.add(componentFileContent);
fileContents.add(new FileContent(processTemplate(new HashMap<>(), CNN_HELPER), CNN_HELPER + ".h"));
fileContents.add(new FileContent(readResource("CNNTranslator.h", Charsets.UTF_8), "CNNTranslator.h"));
}
protected String transformComponent(String component, String predictorClassName, String executeMethod){
......@@ -215,7 +217,7 @@ public class EMADLGenerator {
//insert includes
component = component.replaceFirst("using namespace",
"#include \"" + predictorClassName + ".h" + "\"\n" +
"#include \"" + CNN_HELPER + ".h" + "\"\n" +
"#include \"CNNTranslator.h\"\n" +
"using namespace");
//insert network attribute
......@@ -252,34 +254,28 @@ public class EMADLGenerator {
}
}
public FileContent generateCNNTrainer(Set<ExpandedComponentInstanceSymbol> allInstances, String mainComponentName){
List<ExpandedComponentInstanceSymbol> cnnInstances = new ArrayList<>();
List<String> trainParams = new ArrayList<>();
Set<String> componentNames = new HashSet<>();
for (ExpandedComponentInstanceSymbol componentInstance : allInstances){
public List<FileContent> generateCNNTrainer(Set<ExpandedComponentInstanceSymbol> allInstances, String mainComponentName) {
List<String> cnnInstanceNames = new ArrayList<>();
List<ConfigurationSymbol> configurations = new ArrayList<>();
for (ExpandedComponentInstanceSymbol componentInstance : allInstances) {
ComponentSymbol component = componentInstance.getComponentType().getReferencedSymbol();
Optional<ArchitectureSymbol> architecture = component.getSpannedScope().resolve("", ArchitectureSymbol.KIND);
if (architecture.isPresent()){
String fileContent = getTrainingParamsForComponent(mainComponentName, component, componentInstance);
if (!fileContent.isEmpty()) {
trainParams.add(fileContent);
}
cnnInstances.add(componentInstance);
componentNames.add(component.getFullName());
if (architecture.isPresent()) {
ConfigurationSymbol configuration = getTrainingConfiguration(mainComponentName, component, componentInstance);
configurations.add(configuration);
cnnInstanceNames.add(componentInstance.getFullName().replaceAll("\\.", "_"));
}
}
Map<String, Object> ftlContext = new HashMap<>();
ftlContext.put("instances", cnnInstances);
ftlContext.put("componentNames", componentNames);
ftlContext.put("trainParams", trainParams);
return new FileContent(processTemplate(ftlContext, CNN_TRAINER), CNN_TRAINER + "_" + mainComponentName + ".py");
List<FileContent> fileContents = new ArrayList<>();
Map<String, String> fileContentMap = cnnArchGenerator.generateTrainer(configurations, cnnInstanceNames, mainComponentName);
for (String fileName : fileContentMap.keySet()){
fileContents.add(new FileContent(fileContentMap.get(fileName), fileName));
}
return fileContents;
}
private String getTrainingParamsForComponent(String mainComponentName, ComponentSymbol component, ExpandedComponentInstanceSymbol instance) {
public ConfigurationSymbol getTrainingConfiguration(String mainComponentName, ComponentSymbol component, ExpandedComponentInstanceSymbol instance) {
String configFilename;
String mainComponentConfigFilename = mainComponentName.replaceAll("\\.", "/");
String componentConfigFilename = component.getFullName().replaceAll("\\.", "/");
......@@ -300,7 +296,7 @@ public class EMADLGenerator {
+ getModelsPath() + componentConfigFilename + ".cnnt', '"
+ getModelsPath() + mainComponentConfigFilename + ".cnnt'." +
" These files denote respectively the configuration for the single instance, the component or the whole system.");
return "";
return null;
}
//should be removed when CNNTrain supports packages
......@@ -309,29 +305,32 @@ public class EMADLGenerator {
Path modelPath = Paths.get(getModelsPath() + Joiner.on("/").join(names.subList(0,names.size()-1)));
//
CNNTrainGenerator cnnTrainGenerator = new CNNTrainGenerator();
//CNNTrainGenerator cnnTrainGenerator = new CNNTrainGenerator(); //No need of cnnTrainGenerator since cnnArchGenerator can also generateTrainer()
final ModelPath mp = new ModelPath(modelPath);
GlobalScope trainScope = new GlobalScope(mp, new CNNTrainLanguage());
Map.Entry<String, String> fileContents = cnnTrainGenerator.generateFileContent( trainScope, configFilename );
return fileContents.getValue();
Optional<CNNTrainCompilationUnitSymbol> compilationUnit = trainScope.resolve(configFilename, CNNTrainCompilationUnitSymbol.KIND);
if (!compilationUnit.isPresent()){
Log.error("CNNTrainCompilationUnitSymbol is empty. Could not resolve configuration " + configFilename);
System.exit(1);
}
CNNTrainCocos.checkAll(compilationUnit.get());
ConfigurationSymbol configuration = compilationUnit.get().getConfiguration();
return configuration;
}
protected String processTemplate(Map<String, Object> ftlContext, String templateNameWithoutEnding){
StringWriter writer = new StringWriter();
String templateName = templateNameWithoutEnding + ".ftl";
try{
Template template = TemplateConfiguration.get().getTemplate(templateName);
template.process(ftlContext, writer);
}
catch (IOException e) {
Log.error("Freemarker could not find template " + templateName + " :\n" + e.getMessage());
public String readResource(final String fileName, Charset charset) {
try {
return Resources.toString(Resources.getResource(fileName), charset);
} catch (IllegalArgumentException e) {
System.err.println("Resource file " + fileName + " not found");
System.exit(1);
}
catch (TemplateException e){
Log.error("An exception occured in template " + templateName + " :\n" + e.getMessage());
return null;
} catch (IOException e) {
System.err.println("IO Error occurred");
System.exit(1);
return null;
}
return writer.toString();
}
}
......@@ -26,6 +26,7 @@ import freemarker.template.TemplateException;
import org.apache.commons.cli.*;
import java.io.IOException;
import java.util.Optional;
public class EMADLGeneratorCli {
......@@ -49,6 +50,12 @@ public class EMADLGeneratorCli {
.hasArg(true)
.required(false)
.build();
public static final Option OPTION_BACKEND = Option.builder("b")
.longOpt("backend")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2")
.hasArg(true)
.required(false)
.build();
private EMADLGeneratorCli() {
}
......@@ -67,6 +74,7 @@ public class EMADLGeneratorCli {
options.addOption(OPTION_MODELS_PATH);
options.addOption(OPTION_ROOT_MODEL);
options.addOption(OPTION_OUTPUT_PATH);
options.addOption(OPTION_BACKEND);
return options;
}
......@@ -85,7 +93,21 @@ public class EMADLGeneratorCli {
private static void runGenerator(CommandLine cliArgs) {
String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
String outputPath = cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt());
EMADLGenerator generator = new EMADLGenerator();
String backendString = cliArgs.getOptionValue(OPTION_BACKEND.getOpt());
final String DEFAULT_BACKEND = "MXNET";
if (backendString == null) {
Log.warn("backend not specified. backend set to default value " + DEFAULT_BACKEND);
backendString = DEFAULT_BACKEND;
}
Optional<Backend> backend = Backend.getBackendFromString(backendString);
if (!backend.isPresent()){
Log.warn("specified backend " + backendString + " not supported. backend set to default value " + DEFAULT_BACKEND);
backend = Backend.getBackendFromString(DEFAULT_BACKEND);
}
EMADLGenerator generator = new EMADLGenerator(backend.get());
if (outputPath != null){
generator.setGenerationTargetPath(outputPath);
}
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.generator;
import freemarker.template.Configuration;
import freemarker.template.TemplateExceptionHandler;
public class TemplateConfiguration {
private static TemplateConfiguration instance;
private Configuration configuration;
private TemplateConfiguration() {
configuration = new Configuration(Configuration.VERSION_2_3_23);
configuration.setClassForTemplateLoading(TemplateConfiguration.class, "/templates/");
configuration.setDefaultEncoding("UTF-8");
configuration.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
}
public Configuration getConfiguration() {
return configuration;
}
public static Configuration get(){
if (instance == null){
instance = new TemplateConfiguration();
}
return instance.getConfiguration();
}
}
import logging
import mxnet as mx
<#list instances as instance>
import CNNCreator_${instance.fullName?replace(".", "_")}
</#list>
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
logger.addHandler(handler)
<#list instances as instance>
${instance.fullName?replace(".", "_")} = CNNCreator_${instance.fullName?replace(".", "_")}.CNNCreator_${instance.fullName?replace(".", "_")}()
${instance.fullName?replace(".", "_")}.train(
<#if (trainParams[instance_index])??>
${trainParams[instance_index]}
</#if>
)
</#list>
\ No newline at end of file
......@@ -20,7 +20,7 @@
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.AbstractSymtab;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import org.junit.Assert;
......@@ -36,7 +36,7 @@ import static org.junit.Assert.assertTrue;
public class AbstractSymtabTest {
protected static TaggingResolver createSymTab(String... modelPath) {
return AbstractSymtab.createSymTabAndTaggingResolver(modelPath);
return EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelPath);
}
public static void checkFilesAreEqual(Path generationPath, Path resultsPath, List<String> fileNames) {
......
......@@ -44,7 +44,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testCifar10Generation() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier"};
String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
......@@ -65,7 +65,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testSimulatorGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController"};
String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -73,7 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testAddGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Add"};
String[] args = {"-m", "src/test/resources/models/", "-r", "Add", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -81,7 +81,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testAlexnetGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet"};
String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -89,7 +89,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testResNeXtGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50"};
String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings