Commit 4e6e6673 authored by Svetlana Pavlitskaya's avatar Svetlana Pavlitskaya Committed by Evgeny Kusmenko

Adapt different backends, moved to a newer EMADL version

parent 9c3425c2
......@@ -28,20 +28,23 @@ masterJobLinux:
image: maven:3-jdk-8
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean deploy --settings settings.xml
- cat target/site/jacoco/index.html
- mvn package sonar:sonar -s settings.xml
only:
- master
masterJobWindows:
stage: windows
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
tags:
- Windows10
#masterJobWindows:
# stage: windows
# script:
# - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
# tags:
# - Windows10
BranchJobLinux:
stage: linux
image: maven:3-jdk-8
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
- cat target/site/jacoco/index.html
except:
- master
![pipeline](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/build.svg)
![coverage](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/coverage.svg)
# EMADL2CPP
Generates CPP/Python code for EmbeddedMontiArcDL.
See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/EMADL-Demo) for more information on how the generated code can be used.
\ No newline at end of file
......@@ -8,17 +8,18 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.2.1-SNAPSHOT</version>
<version>0.2.2-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<emadl.version>0.2.1-SNAPSHOT</emadl.version>
<CNNTrain.version>0.2.1-SNAPSHOT</CNNTrain.version>
<cnnarch-generator.version>0.2.1-SNAPSHOT</cnnarch-generator.version>
<embedded-montiarc-math-generator>0.0.10</embedded-montiarc-math-generator>
<emadl.version>0.2.2-SNAPSHOT</emadl.version>
<CNNTrain.version>0.2.4-SNAPSHOT</CNNTrain.version>
<cnnarch-mxnet-generator.version>0.2.3-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.2-SNAPSHOT</cnnarch-caffe2-generator.version>
<embedded-montiarc-math-generator>0.0.25-SNAPSHOT</embedded-montiarc-math-generator>
<!-- .. Libraries .................................................. -->
<guava.version>18.0</guava.version>
......@@ -31,6 +32,7 @@
<compiler.plugin>3.3</compiler.plugin>
<source.plugin>2.4</source.plugin>
<shade.plugin>2.4.3</shade.plugin>
<jacoco.plugin>0.8.1</jacoco.plugin>
<!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier>
......@@ -68,7 +70,13 @@
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId>
<version>${cnnarch-generator.version}</version>
<version>${cnnarch-mxnet-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-caffe2-generator</artifactId>
<version>${cnnarch-caffe2-generator.version}</version>
</dependency>
<dependency>
......@@ -132,6 +140,27 @@
<version>2.8.1</version>
</plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
......
......@@ -57,6 +57,24 @@
</mirrors>
<profiles>
<profile>
<id>sonar</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<!-- Optional URL to server. Default value is http://localhost:9000 -->
<sonar.host.url>
https://metric.se.rwth-aachen.de
</sonar.host.url>
<sonar.login>
jenkins
</sonar.login>
<sonar.password>
${env.sonar}
</sonar.password>
</properties>
</profile>
<profile>
<id>se-nexus</id>
......
package de.monticore.lang.monticar.emadl.generator;
import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
import java.util.Optional;
public enum Backend {
MXNET{
@Override
public CNNArchGenerator getGenerator() {
return new CNNArch2MxNet();
}
},
CAFFE2{
@Override
public CNNArchGenerator getGenerator() {
return new CNNArch2Caffe2();
}
};
public abstract CNNArchGenerator getGenerator();
public static Optional<Backend> getBackendFromString(String backend){
switch (backend){
case "MXNET":
return Optional.of(MXNET);
case "CAFFE2":
return Optional.of(CAFFE2);
default:
return Optional.empty();
}
}
}
......@@ -44,7 +44,10 @@ import de.monticore.symboltable.Scope;
import java.nio.file.Paths;
import java.util.Arrays;
public class AbstractSymtab {
public class EMADLAbstractSymtab {
public EMADLAbstractSymtab() {
}
public static TaggingResolver createSymTabAndTaggingResolver(String... modelPath) {
Scope scope = createSymTab(modelPath);
TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));
......
......@@ -26,6 +26,7 @@ import freemarker.template.TemplateException;
import org.apache.commons.cli.*;
import java.io.IOException;
import java.util.Optional;
public class EMADLGeneratorCli {
......@@ -49,6 +50,12 @@ public class EMADLGeneratorCli {
.hasArg(true)
.required(false)
.build();
public static final Option OPTION_BACKEND = Option.builder("b")
.longOpt("backend")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2")
.hasArg(true)
.required(false)
.build();
private EMADLGeneratorCli() {
}
......@@ -67,6 +74,7 @@ public class EMADLGeneratorCli {
options.addOption(OPTION_MODELS_PATH);
options.addOption(OPTION_ROOT_MODEL);
options.addOption(OPTION_OUTPUT_PATH);
options.addOption(OPTION_BACKEND);
return options;
}
......@@ -85,7 +93,21 @@ public class EMADLGeneratorCli {
private static void runGenerator(CommandLine cliArgs) {
String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
String outputPath = cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt());
EMADLGenerator generator = new EMADLGenerator();
String backendString = cliArgs.getOptionValue(OPTION_BACKEND.getOpt());
final String DEFAULT_BACKEND = "MXNET";
if (backendString == null) {
Log.warn("backend not specified. backend set to default value " + DEFAULT_BACKEND);
backendString = DEFAULT_BACKEND;
}
Optional<Backend> backend = Backend.getBackendFromString(backendString);
if (!backend.isPresent()){
Log.warn("specified backend " + backendString + " not supported. backend set to default value " + DEFAULT_BACKEND);
backend = Backend.getBackendFromString(DEFAULT_BACKEND);
}
EMADLGenerator generator = new EMADLGenerator(backend.get());
if (outputPath != null){
generator.setGenerationTargetPath(outputPath);
}
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.generator;
import freemarker.template.Configuration;
import freemarker.template.TemplateExceptionHandler;
public class TemplateConfiguration {
private static TemplateConfiguration instance;
private Configuration configuration;
private TemplateConfiguration() {
configuration = new Configuration(Configuration.VERSION_2_3_23);
configuration.setClassForTemplateLoading(TemplateConfiguration.class, "/templates/");
configuration.setDefaultEncoding("UTF-8");
configuration.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
}
public Configuration getConfiguration() {
return configuration;
}
public static Configuration get(){
if (instance == null){
instance = new TemplateConfiguration();
}
return instance.getConfiguration();
}
}
import logging
import mxnet as mx
<#list instances as instance>
import CNNCreator_${instance.fullName?replace(".", "_")}
</#list>
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
logger.addHandler(handler)
<#list instances as instance>
${instance.fullName?replace(".", "_")} = CNNCreator_${instance.fullName?replace(".", "_")}.CNNCreator_${instance.fullName?replace(".", "_")}()
${instance.fullName?replace(".", "_")}.train(
<#if (trainParams[instance_index])??>
${trainParams[instance_index]}
</#if>
)
</#list>
\ No newline at end of file
......@@ -20,7 +20,7 @@
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.AbstractSymtab;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import org.junit.Assert;
......@@ -36,7 +36,7 @@ import static org.junit.Assert.assertTrue;
public class AbstractSymtabTest {
protected static TaggingResolver createSymTab(String... modelPath) {
return AbstractSymtab.createSymTabAndTaggingResolver(modelPath);
return EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelPath);
}
public static void checkFilesAreEqual(Path generationPath, Path resultsPath, List<String> fileNames) {
......
......@@ -44,7 +44,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testCifar10Generation() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier"};
String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
......@@ -65,7 +65,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testSimulatorGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController"};
String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -73,7 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testAddGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Add"};
String[] args = {"-m", "src/test/resources/models/", "-r", "Add", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -81,7 +81,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testAlexnetGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet"};
String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -89,7 +89,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testResNeXtGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50"};
String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -97,7 +97,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testThreeInputGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14"};
String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().size() == 1);
}
......@@ -105,7 +105,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testMultipleOutputsGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleOutputs"};
String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleOutputs", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().size() == 1);
}
......@@ -113,7 +113,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void tesVGGGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "VGG16"};
String[] args = {"-m", "src/test/resources/models/", "-r", "VGG16", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......@@ -121,7 +121,7 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testMultipleInstances() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "InstanceTest.MainB"};
String[] args = {"-m", "src/test/resources/models/", "-r", "InstanceTest.MainB", "-b", "MXNET"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
......
......@@ -26,7 +26,6 @@ import de.monticore.lang.monticar.emadl._parser.EMADLParser;
import de.monticore.symboltable.Scope;
import de.se_rwth.commons.logging.Log;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertNotNull;
......@@ -48,7 +47,6 @@ public class SymtabTest extends AbstractSymtabTest {
}
@Test
@Ignore
public void testAlexnet(){
Scope symTab = createSymTab("src/test/resources/models");
ComponentSymbol a = symTab.<ComponentSymbol>resolve("ResNet34", ComponentSymbol.KIND).orElse(null);
......
......@@ -19,7 +19,4 @@ component MainB{
connect net2.predictions -> outCalc2.probabilities;
connect outCalc1.digit -> digit1;
connect outCalc2.digit -> digit2;
}
}
\ No newline at end of file
......@@ -13,6 +13,4 @@ component Cifar10Classifier{
connect image -> net.data;
connect net.softmax -> calculateClass.inputVector;
connect calculateClass.maxIndex -> classIndex;
}
}
\ No newline at end of file
......@@ -105,8 +105,9 @@ class CNNCreator_cifar10_cifar10Classifier_net:
sys.exit(1)
def train(self, batch_size,
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
optimizer='adam',
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
......@@ -158,6 +159,7 @@ class CNNCreator_cifar10_cifar10Classifier_net:
self.module.fit(
train_data=train_iter,
eval_metric=eval_metric,
eval_data=test_iter,
optimizer=optimizer,
optimizer_params=optimizer_params,
......
......@@ -29,7 +29,7 @@ public:
}
void predict(const std::vector<float> &data,
std::vector<float> &softmax){
std::vector<float> &softmax){
MXPredSetInput(handle, "data", data.data(), data.size());
//MXPredSetInput(handle, "data", data.data(), data.size());
......
......@@ -5,24 +5,20 @@ import CNNCreator_cifar10_cifar10Classifier_net
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
handler = logging.FileHandler("train.log", "w", encoding=None, delay="true")
logger.addHandler(handler)
cifar10_cifar10Classifier_net = CNNCreator_cifar10_cifar10Classifier_net.CNNCreator_cifar10_cifar10Classifier_net()
cifar10_cifar10Classifier_net.train(
batch_size = 64,
num_epoch = 10,
load_checkpoint = False,
context = 'gpu',
normalize = True,
optimizer = 'adam',
optimizer_params = {
'weight_decay': 1.0E-4,
'learning_rate': 0.01,
'learning_rate_decay': 0.8,
'step_size': 1000}
batch_size=64,
num_epoch=10,
load_checkpoint=False,
context='gpu',
normalize=True,
optimizer='adam',
optimizer_params={
'weight_decay': 1.0E-4,
'learning_rate': 0.01,
'learning_rate_decay': 0.8,
'step_size': 1000}
)
......@@ -9,13 +9,13 @@
using namespace arma;
class cifar10_cifar10Classifier{
public:
cube image;
double classIndex;
icube image;
int classIndex;
cifar10_cifar10Classifier_net net;
cifar10_cifar10Classifier_calculateClass calculateClass;
void init()
{
image = cube(3, 32, 32);
image = icube(3, 32, 32);
net.init();
calculateClass.init();
}
......
......@@ -9,7 +9,7 @@ class cifar10_cifar10Classifier_calculateClass{
const int n = 10;
public:
colvec inputVector;
double maxIndex;
int maxIndex;
void init()
{
inputVector=colvec(n);
......@@ -17,11 +17,11 @@ inputVector=colvec(n);
void execute()
{
maxIndex = 0;
double maxValue = inputVector(0);
double maxValue = inputVector(0-1);
for( auto i=2;i<=n;++i){
if((inputVector(i) > maxValue)){
if((inputVector(i-1) > maxValue)){
maxIndex = i-1;
maxValue = inputVector(i-1) ;
maxValue = inputVector(i-1);
}
}
}
......
......@@ -11,11 +11,11 @@ class cifar10_cifar10Classifier_net{
const int classes = 10;
public:
CNNPredictor_cifar10_cifar10Classifier_net _cnn_;
cube data;
icube data;
colvec softmax;
void init()
{
data = cube(3, 32, 32);
data = icube(3, 32, 32);
softmax=colvec(classes);
}
void execute(){
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment