Commit d8a0221f authored by Abdallah Atouani's avatar Abdallah Atouani

Merge remote-tracking branch 'origin/develop' into feature/path_tagging

# Conflicts:
#	pom.xml
#	src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGenerator.java
#	src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java
parents cbeed5e9 d9d9cf5d
...@@ -5,6 +5,6 @@ nppBackup ...@@ -5,6 +5,6 @@ nppBackup
.classpath .classpath
.idea .idea
.git .git
.vscode
*.iml *.iml
train.log
...@@ -24,7 +24,8 @@ stages: ...@@ -24,7 +24,8 @@ stages:
- linux - linux
- deploy - deploy
masterJobLinux:
git masterJobLinux:
stage: deploy stage: deploy
image: maven:3-jdk-8 image: maven:3-jdk-8
script: script:
...@@ -34,6 +35,7 @@ masterJobLinux: ...@@ -34,6 +35,7 @@ masterJobLinux:
only: only:
- master - master
integrationMXNetJobLinux: integrationMXNetJobLinux:
stage: linux stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3 image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
...@@ -47,6 +49,7 @@ integrationCaffe2JobLinux: ...@@ -47,6 +49,7 @@ integrationCaffe2JobLinux:
script: script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationCaffe2Test - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationCaffe2Test
integrationGluonJobLinux: integrationGluonJobLinux:
stage: linux stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3 image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
...@@ -54,10 +57,17 @@ integrationGluonJobLinux: ...@@ -54,10 +57,17 @@ integrationGluonJobLinux:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationGluonTest - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationGluonTest
integrationPythonWrapperTest:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2pythonwrapper/tests/mvn-swig:latest
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationPythonWrapperTest
masterJobWindows: masterJobWindows:
stage: windows stage: windows
script: script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest="GenerationTest,SymtabTest" - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B -U clean install --settings settings.xml -Dtest="GenerationTest,SymtabTest"
tags: tags:
- Windows10 - Windows10
......
{
"configurations": [
{
"type": "java",
"name": "CodeLens (Launch) - EMADLGeneratorCli",
"request": "launch",
"mainClass": "de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli",
"projectName": "embedded-montiarc-emadl-generator"
}
]
}
\ No newline at end of file
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
<groupId>de.monticore.lang.monticar</groupId> <groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId> <artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.3.0</version> <version>0.3.4-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= --> <!-- == PROJECT DEPENDENCIES ============================================= -->
...@@ -16,10 +16,11 @@ ...@@ -16,10 +16,11 @@
<!-- .. SE-Libraries .................................................. --> <!-- .. SE-Libraries .................................................. -->
<emadl.version>0.2.9-SNAPSHOT</emadl.version> <emadl.version>0.2.9-SNAPSHOT</emadl.version>
<CNNTrain.version>0.2.6</CNNTrain.version> <CNNTrain.version>0.3.6-SNAPSHOT</CNNTrain.version>
<cnnarch-mxnet-generator.version>0.2.14-SNAPSHOT</cnnarch-mxnet-generator.version> <cnnarch-generator.version>0.0.2-SNAPSHOT</cnnarch-generator.version>
<cnnarch-caffe2-generator.version>0.2.11-SNAPSHOT</cnnarch-caffe2-generator.version> <cnnarch-mxnet-generator.version>0.2.16-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-gluon-generator.version>0.1.6</cnnarch-gluon-generator.version> <cnnarch-caffe2-generator.version>0.2.12-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.2.5-SNAPSHOT</cnnarch-gluon-generator.version>
<embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator> <embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
<!-- .. Libraries .................................................. --> <!-- .. Libraries .................................................. -->
...@@ -73,6 +74,12 @@ ...@@ -73,6 +74,12 @@
<version>${embedded-montiarc-math-opt-generator}</version> <version>${embedded-montiarc-math-opt-generator}</version>
</dependency> </dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-generator</artifactId>
<version>${cnnarch-generator.version}</version>
</dependency>
<dependency> <dependency>
<groupId>de.monticore.lang.monticar</groupId> <groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId> <artifactId>cnnarch-mxnet-generator</artifactId>
......
package de.monticore.lang.monticar.emadl.generator; package de.monticore.lang.monticar.emadl.generator;
import de.monticore.lang.monticar.cnnarch.CNNArchGenerator; import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2Gluon; import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2Gluon;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNTrain2Gluon; import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNTrain2Gluon;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet; import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2; import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNTrain2MxNet; import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNTrain2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNTrain2Caffe2; import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNTrain2Caffe2;
import de.monticore.lang.monticar.cnntrain.CNNTrainGenerator; import de.monticore.lang.monticar.emadl.generator.reinforcementlearning.RewardFunctionCppGenerator;
import java.util.Optional; import java.util.Optional;
...@@ -40,7 +41,7 @@ public enum Backend { ...@@ -40,7 +41,7 @@ public enum Backend {
} }
@Override @Override
public CNNTrainGenerator getCNNTrainGenerator() { public CNNTrainGenerator getCNNTrainGenerator() {
return new CNNTrain2Gluon(); return new CNNTrain2Gluon(new RewardFunctionCppGenerator());
} }
}; };
......
...@@ -52,7 +52,7 @@ public class EMADLGeneratorCli { ...@@ -52,7 +52,7 @@ public class EMADLGeneratorCli {
.build(); .build();
public static final Option OPTION_BACKEND = Option.builder("b") public static final Option OPTION_BACKEND = Option.builder("b")
.longOpt("backend") .longOpt("backend")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2") .desc("deep-learning-framework backend. Options: MXNET, CAFFE2, GLUON")
.hasArg(true) .hasArg(true)
.required(false) .required(false)
.build(); .build();
......
package de.monticore.lang.monticar.emadl.generator.reinforcementlearning;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.cnnarch.gluongenerator.reinforcement.RewardFunctionSourceGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.monticar.generator.cpp.GeneratorEMAMOpt2CPP;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import java.io.IOException;
import java.util.Optional;
public class RewardFunctionCppGenerator implements RewardFunctionSourceGenerator{
public RewardFunctionCppGenerator() {
}
@Override
public EMAComponentInstanceSymbol resolveSymbol(TaggingResolver taggingResolver, String rootModel) {
Optional<EMAComponentInstanceSymbol> instanceSymbol = taggingResolver
.<EMAComponentInstanceSymbol>resolve(rootModel, EMAComponentInstanceSymbol.KIND);
if (!instanceSymbol.isPresent()) {
Log.error("Generation of reward function is not possible: Cannot resolve component instance "
+ rootModel);
}
return instanceSymbol.get();
}
@Override
public void generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver,
String targetPath) {
GeneratorEMAMOpt2CPP generator = new GeneratorEMAMOpt2CPP();
generator.useArmadilloBackend();
generator.setGenerationTargetPath(targetPath);
try {
generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) {
Log.error("Generation of reward function is not possible: " + e.getMessage());
}
}
@Override
public void generate(String modelPath, String rootModel, String targetPath) {
TaggingResolver taggingResolver = createTaggingResolver(modelPath);
EMAComponentInstanceSymbol instanceSymbol = resolveSymbol(taggingResolver, rootModel);
generate(instanceSymbol, taggingResolver, targetPath);
}
@Override
public TaggingResolver createTaggingResolver(final String modelPath) {
return EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelPath);
}
}
...@@ -20,25 +20,9 @@ ...@@ -20,25 +20,9 @@
*/ */
package de.monticore.lang.monticar.emadl; package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend; import org.junit.Ignore;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
@Ignore
public class IntegrationCaffe2Test extends IntegrationTest { public class IntegrationCaffe2Test extends IntegrationTest {
public IntegrationCaffe2Test() { public IntegrationCaffe2Test() {
super("CAFFE2", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#13D139510DC5681639AA91D7250288D3#1A42D4842D0664937A9F6B727BD60CEF"); super("CAFFE2", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#13D139510DC5681639AA91D7250288D3#1A42D4842D0664937A9F6B727BD60CEF");
......
...@@ -20,8 +20,48 @@ ...@@ -20,8 +20,48 @@
*/ */
package de.monticore.lang.monticar.emadl; package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import org.junit.Ignore;
import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class IntegrationGluonTest extends IntegrationTest { public class IntegrationGluonTest extends IntegrationTest {
private Path multipleStreamsHashFile = Paths.get("./target/generated-sources-emadl/MultipleStreams.training_hash");
public IntegrationGluonTest() { public IntegrationGluonTest() {
super("GLUON", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#C4C23549E737A759721D6694C75D9771#5AF0CE68E408E8C1F000E49D72AC214A"); super("GLUON", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#C4C23549E737A759721D6694C75D9771#5AF0CE68E408E8C1F000E49D72AC214A");
} }
@Test
public void testMultipleStreams() {
Log.getFindings().clear();
deleteHashFile(multipleStreamsHashFile);
String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleStreams", "-b", "GLUON"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
private void deleteHashFile(Path hashFile) {
try {
Files.delete(hashFile);
}
catch (NoSuchFileException e) {
}
catch(Exception e) {
assertFalse("Could not delete hash file", true);
}
}
} }
...@@ -20,24 +20,7 @@ ...@@ -20,24 +20,7 @@
*/ */
package de.monticore.lang.monticar.emadl; package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend; import org.junit.Ignore;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class IntegrationMXNetTest extends IntegrationTest { public class IntegrationMXNetTest extends IntegrationTest {
public IntegrationMXNetTest() { public IntegrationMXNetTest() {
......
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import org.junit.Ignore;
import org.junit.Test;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.stream.Collectors;
import static junit.framework.TestCase.assertTrue;
/**
*
*/
public class IntegrationPythonWrapperTest extends AbstractSymtabTest {
@Test
public void testGluonReinforcementModelRosEnvironment() {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/reinforcementModel", "-r", "torcs.agent.TorcsAgent", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().stream().filter(Finding::isError).collect(Collectors.toList()).isEmpty());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/torcs"),
Arrays.asList(
"CMakeLists.txt",
"CNNBufferFile.h",
"torcs_agent_torcsAgent.cpp",
"torcs_agent_torcsAgent.h",
"torcs_agent_torcsAgent_dqn.h",
"torcs_agent_torcsAgent_policy.h",
"CNNCreator_torcs_agent_torcsAgent_dqn.py",
"CNNNet_torcs_agent_torcsAgent_dqn.py",
"CNNPredictor_torcs_agent_torcsAgent_dqn.h",
"CNNTrainer_torcs_agent_torcsAgent_dqn.py",
"CNNTranslator.h",
"HelperA.h",
"start_training.sh",
"reward/CMakeLists.txt",
"reward/HelperA.h",
"reward/torcs_agent_dqn_reward.cpp",
"reward/torcs_agent_dqn_reward.h",
"reward/pylib/CMakeLists.txt",
"reward/pylib/torcs_agent_dqn_reward_executor.cpp",
"reward/pylib/torcs_agent_dqn_reward_executor.h",
"reward/pylib/torcs_agent_dqn_reward_executor.i",
"reward/pylib/armanpy/armanpy.hpp",
"reward/pylib/armanpy/armanpy.i",
"reward/pylib/armanpy/armanpy_1d.i",
"reward/pylib/armanpy/armanpy_2d.i",
"reward/pylib/armanpy/armanpy_3d.i",
"reward/pylib/armanpy/numpy.i",
"reinforcement_learning/__init__.py",
"reinforcement_learning/strategy.py",
"reinforcement_learning/agent.py",
"reinforcement_learning/environment.py",
"reinforcement_learning/replay_memory.py",
"reinforcement_learning/util.py",
"reinforcement_learning/cnnarch_logger.py"
)
);
assertTrue(Paths.get(
"./target/generated-sources-emadl/reinforcement_learning/_torcs_agent_dqn_reward_executor.so")
.toFile().exists());
assertTrue(Paths.get(
"./target/generated-sources-emadl/reinforcement_learning/torcs_agent_dqn_reward_executor.py")
.toFile().exists());
}
}
\ No newline at end of file
...@@ -26,10 +26,7 @@ import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli; ...@@ -26,10 +26,7 @@ import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log; import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException; import freemarker.template.TemplateException;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.junit.AfterClass; import org.junit.*;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
......
...@@ -41,7 +41,7 @@ component Alexnet{ ...@@ -41,7 +41,7 @@ component Alexnet{
fc(->=2) -> fc(->=2) ->
FullyConnected(units=1000) -> FullyConnected(units=1000) ->
Softmax() -> Softmax() ->
predictions predictions;
} }
} }
\ No newline at end of file
...@@ -18,7 +18,7 @@ component NetworkB<Z classes = 10> (Z convChannels){ ...@@ -18,7 +18,7 @@ component NetworkB<Z classes = 10> (Z convChannels){
Dropout() -> Dropout() ->
FullyConnected(units=classes) -> FullyConnected(units=classes) ->
Softmax() -> Softmax() ->
predictions predictions;
} }
} }
\ No newline at end of file
...@@ -27,6 +27,6 @@ component InvalidLayerInput{ ...@@ -27,6 +27,6 @@ component InvalidLayerInput{
fc() -> fc() ->
FullyConnected(units=1000) -> FullyConnected(units=1000) ->
Softmax() -> Softmax() ->
predictions predictions;
} }
} }
\ No newline at end of file
configuration Invariant{
num_epoch:10
batch_size:5
context:cpu
optimizer:adam{
learning_rate:0.01
learning_rate_decay:0.8
step_size:1000
weight_decay:0.0001
}
}
component Invariant{
ports in Z(0:3)^{1} data[2],
out Q(0:1)^{4} pred[3];
implementation CNN {
data[0] ->
FullyConnected(units=4) ->
Softmax() ->
pred[0];
data[1] ->
OneHot(size=4) ->
pred[1];
1 ->
OneHot(size=4) ->
pred[2];
}
}
\ No newline at end of file
configuration MultipleInputs{
num_epoch:10
batch_size:5
context:cpu
optimizer:adam{