Skip to content

GitLab

E
EMADL2CPP
Commit 8a07e768 authored by Nicola Gatto's avatar Nicola Gatto Committed by Evgeny Kusmenko
Browse files
Options

Fixed some tests and removed MultipleInput/MultipleOutput test since the... 

Fixed some tests and removed MultipleInput/MultipleOutput test since the functionality is already covered in MultipleStreams test
parent 16023357
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 16537 additions and 247 deletions
.gitignore
View file @ 8a07e768
......@@ -5,6 +5,6 @@ nppBackup
.classpath
.idea
.git
.vscode
*.iml
train.log
.gitlab-ci.yml
View file @ 8a07e768
......@@ -24,7 +24,8 @@ stages:
- linux
- deploy
masterJobLinux:
git masterJobLinux:
stage: deploy
image: maven:3-jdk-8
script:
......@@ -34,6 +35,7 @@ masterJobLinux:
only:
- master
integrationMXNetJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
......@@ -47,17 +49,25 @@ integrationCaffe2JobLinux:
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationCaffe2Test
integrationGluonJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationGluonTest
integrationPythonWrapperTest:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2pythonwrapper/tests/mvn-swig:latest
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationPythonWrapperTest
masterJobWindows:
stage: windows
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest="GenerationTest,SymtabTest"
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B -U clean install --settings settings.xml -Dtest="GenerationTest,SymtabTest"
tags:
- Windows10
......
.vscode/launch.json deleted 100644 → 0
View file @ 16023357
{
"configurations": [
{
"type": "java",
"name": "CodeLens (Launch) - EMADLGeneratorCli",
"request": "launch",
"mainClass": "de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli",
"projectName": "embedded-montiarc-emadl-generator"
}
]
}
\ No newline at end of file
pom.xml
View file @ 8a07e768
......@@ -8,18 +8,19 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.3.0</version>
<version>0.3.5-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<emadl.version>0.2.6</emadl.version>
<CNNTrain.version>0.2.6</CNNTrain.version>
<cnnarch-mxnet-generator.version>0.2.14-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.11-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.1.6</cnnarch-gluon-generator.version>
<emadl.version>0.2.8-SNAPSHOT</emadl.version>
<CNNTrain.version>0.3.6-SNAPSHOT</CNNTrain.version>
<cnnarch-generator.version>0.0.2-SNAPSHOT</cnnarch-generator.version>
<cnnarch-mxnet-generator.version>0.2.16-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.12-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.2.6-SNAPSHOT</cnnarch-gluon-generator.version>
<embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
<!-- .. Libraries .................................................. -->
......@@ -68,6 +69,12 @@
<version>${embedded-montiarc-math-opt-generator}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-generator</artifactId>
<version>${cnnarch-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId>
......
src/main/java/de/monticore/lang/monticar/emadl/generator/Backend.java
View file @ 8a07e768
package de.monticore.lang.monticar.emadl.generator;
import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2Gluon;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNTrain2Gluon;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNTrain2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNTrain2Caffe2;
import de.monticore.lang.monticar.cnntrain.CNNTrainGenerator;
import de.monticore.lang.monticar.emadl.generator.reinforcementlearning.RewardFunctionCppGenerator;
import java.util.Optional;
......@@ -40,7 +41,7 @@ public enum Backend {
}
@Override
public CNNTrainGenerator getCNNTrainGenerator() {
return new CNNTrain2Gluon();
return new CNNTrain2Gluon(new RewardFunctionCppGenerator());
}
};
......
src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGeneratorCli.java
View file @ 8a07e768
......@@ -52,7 +52,7 @@ public class EMADLGeneratorCli {
.build();
public static final Option OPTION_BACKEND = Option.builder("b")
.longOpt("backend")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2, GLUON")
.hasArg(true)
.required(false)
.build();
......
src/main/java/de/monticore/lang/monticar/emadl/generator/reinforcementlearning/RewardFunctionCppGenerator.java 0 → 100644
View file @ 8a07e768
package de.monticore.lang.monticar.emadl.generator.reinforcementlearning;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.cnnarch.gluongenerator.reinforcement.RewardFunctionSourceGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.monticar.generator.cpp.GeneratorEMAMOpt2CPP;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import java.io.IOException;
import java.util.Optional;
public class RewardFunctionCppGenerator implements RewardFunctionSourceGenerator{
public RewardFunctionCppGenerator() {
}
@Override
public EMAComponentInstanceSymbol resolveSymbol(TaggingResolver taggingResolver, String rootModel) {
Optional<EMAComponentInstanceSymbol> instanceSymbol = taggingResolver
.<EMAComponentInstanceSymbol>resolve(rootModel, EMAComponentInstanceSymbol.KIND);
if (!instanceSymbol.isPresent()) {
Log.error("Generation of reward function is not possible: Cannot resolve component instance "
+ rootModel);
}
return instanceSymbol.get();
}
@Override
public void generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver,
String targetPath) {
GeneratorEMAMOpt2CPP generator = new GeneratorEMAMOpt2CPP();
generator.useArmadilloBackend();
generator.setGenerationTargetPath(targetPath);
try {
generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) {
Log.error("Generation of reward function is not possible: " + e.getMessage());
}
}
@Override
public void generate(String modelPath, String rootModel, String targetPath) {
TaggingResolver taggingResolver = createTaggingResolver(modelPath);
EMAComponentInstanceSymbol instanceSymbol = resolveSymbol(taggingResolver, rootModel);
generate(instanceSymbol, taggingResolver, targetPath);
}
@Override
public TaggingResolver createTaggingResolver(final String modelPath) {
return EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelPath);
}
}
src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java
View file @ 8a07e768
......@@ -23,9 +23,11 @@ package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
......@@ -35,12 +37,13 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class GenerationTest extends AbstractSymtabTest {
@Before
public void setUp() {
// ensure an empty log
......@@ -140,7 +143,7 @@ public class GenerationTest extends AbstractSymtabTest {
}
@Test
public void testMnistClassifier() throws IOException, TemplateException {
public void testMnistClassifierForCaffe2() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "mnist.MnistClassifier", "-b", "CAFFE2", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
......@@ -178,7 +181,7 @@ public class GenerationTest extends AbstractSymtabTest {
"CNNCreator_mnist_mnistClassifier_net.py",
"CNNPredictor_mnist_mnistClassifier_net.h",
"CNNDataLoader_mnist_mnistClassifier_net.py",
"supervised_trainer.py",
"CNNSupervisedTrainer_mnist_mnistClassifier_net.py",
"mnist_mnistClassifier_net.h",
"HelperA.h",
"CNNTranslator.h",
......@@ -187,6 +190,48 @@ public class GenerationTest extends AbstractSymtabTest {
"mnist_mnistClassifier_net.h"));
}
@Test
public void testInvariantForGluon() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Invariant", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().size() == 0);
}
@Test
public void testGluonReinforcementModelGymEnvironment() {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/reinforcementModel", "-r", "cartpole.Master", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().stream().filter(Finding::isError).collect(Collectors.toList()).isEmpty());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/cartpole"),
Arrays.asList(
"cartpole_master.cpp",
"cartpole_master.h",
"cartpole_master_dqn.h",
"cartpole_master_policy.h",
"CMakeLists.txt",
"CNNBufferFile.h",
"CNNCreator_cartpole_master_dqn.py",
"CNNNet_cartpole_master_dqn.py",
"CNNPredictor_cartpole_master_dqn.h",
"CNNTrainer_cartpole_master_dqn.py",
"CNNTranslator.h",
"HelperA.h",
"start_training.sh",
"reinforcement_learning/__init__.py",
"reinforcement_learning/strategy.py",
"reinforcement_learning/agent.py",
"reinforcement_learning/environment.py",
"reinforcement_learning/replay_memory.py",
"reinforcement_learning/util.py",
"reinforcement_learning/cnnarch_logger.py"
)
);
}
@Test
public void testHashFunction() {
EMADLGenerator tester = new EMADLGenerator(Backend.MXNET);
......@@ -197,4 +242,40 @@ public class GenerationTest extends AbstractSymtabTest {
} catch(IOException e){
}
}
@Test
public void gluonDdpgTest() {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/reinforcementModel", "-r", "mountaincar.Master", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertEquals(0, Log.getFindings().stream().filter(Finding::isError).count());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/mountaincar"),
Arrays.asList(
"mountaincar_master.cpp",
"mountaincar_master.h",
"mountaincar_master_actor.h",
"CMakeLists.txt",
"CNNBufferFile.h",
"CNNCreator_mountaincar_master_actor.py",
"CNNNet_mountaincar_master_actor.py",
"CNNPredictor_mountaincar_master_actor.h",
"CNNTrainer_mountaincar_master_actor.py",
"CNNTranslator.h",
"HelperA.h",
"start_training.sh",
"reinforcement_learning/__init__.py",
"reinforcement_learning/CNNCreator_mountaincar_agent_mountaincarCritic.py",
"reinforcement_learning/CNNNet_mountaincar_agent_mountaincarCritic.py",
"reinforcement_learning/strategy.py",
"reinforcement_learning/agent.py",
"reinforcement_learning/environment.py",
"reinforcement_learning/replay_memory.py",
"reinforcement_learning/util.py",
"reinforcement_learning/cnnarch_logger.py"
)
);
}
}
src/test/java/de/monticore/lang/monticar/emadl/IntegrationCaffe2Test.java
View file @ 8a07e768
......@@ -20,27 +20,11 @@
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
import org.junit.Ignore;
@Ignore
public class IntegrationCaffe2Test extends IntegrationTest {
public IntegrationCaffe2Test() {
super("CAFFE2", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#13D139510DC5681639AA91D7250288D3#1A42D4842D0664937A9F6B727BD60CEF");
}
}
}
\ No newline at end of file
src/test/java/de/monticore/lang/monticar/emadl/IntegrationGluonTest.java
View file @ 8a07e768
......@@ -20,8 +20,48 @@
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import org.junit.Ignore;
import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class IntegrationGluonTest extends IntegrationTest {
private Path multipleStreamsHashFile = Paths.get("./target/generated-sources-emadl/MultipleStreams.training_hash");
public IntegrationGluonTest() {
super("GLUON", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#C4C23549E737A759721D6694C75D9771#5AF0CE68E408E8C1F000E49D72AC214A");
}
@Test
public void testMultipleStreams() {
Log.getFindings().clear();
deleteHashFile(multipleStreamsHashFile);
String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleStreams", "-b", "GLUON"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
private void deleteHashFile(Path hashFile) {
try {
Files.delete(hashFile);
}
catch (NoSuchFileException e) {
}
catch(Exception e) {
assertFalse("Could not delete hash file", true);
}
}
}
src/test/java/de/monticore/lang/monticar/emadl/IntegrationMXNetTest.java
View file @ 8a07e768
......@@ -20,24 +20,7 @@
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
import org.junit.Ignore;
public class IntegrationMXNetTest extends IntegrationTest {
public IntegrationMXNetTest() {
......
src/test/java/de/monticore/lang/monticar/emadl/IntegrationPythonWrapperTest.java 0 → 100644
View file @ 8a07e768
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import org.junit.Ignore;
import org.junit.Test;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.stream.Collectors;
import static junit.framework.TestCase.assertTrue;
/**
*
*/
public class IntegrationPythonWrapperTest extends AbstractSymtabTest {
@Test
public void testGluonReinforcementModelRosEnvironment() {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/reinforcementModel", "-r", "torcs.agent.TorcsAgent", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().stream().filter(Finding::isError).collect(Collectors.toList()).isEmpty());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/torcs"),
Arrays.asList(
"CMakeLists.txt",
"CNNBufferFile.h",
"torcs_agent_torcsAgent.cpp",
"torcs_agent_torcsAgent.h",
"torcs_agent_torcsAgent_dqn.h",
"torcs_agent_torcsAgent_policy.h",
"CNNCreator_torcs_agent_torcsAgent_dqn.py",
"CNNNet_torcs_agent_torcsAgent_dqn.py",
"CNNPredictor_torcs_agent_torcsAgent_dqn.h",
"CNNTrainer_torcs_agent_torcsAgent_dqn.py",
"CNNTranslator.h",
"HelperA.h",
"start_training.sh",
"reward/CMakeLists.txt",
"reward/HelperA.h",
"reward/torcs_agent_dqn_reward.cpp",
"reward/torcs_agent_dqn_reward.h",
"reward/pylib/CMakeLists.txt",
"reward/pylib/torcs_agent_dqn_reward_executor.cpp",
"reward/pylib/torcs_agent_dqn_reward_executor.h",
"reward/pylib/torcs_agent_dqn_reward_executor.i",
"reward/pylib/armanpy/armanpy.hpp",
"reward/pylib/armanpy/armanpy.i",
"reward/pylib/armanpy/armanpy_1d.i",
"reward/pylib/armanpy/armanpy_2d.i",
"reward/pylib/armanpy/armanpy_3d.i",
"reward/pylib/armanpy/numpy.i",
"reinforcement_learning/__init__.py",
"reinforcement_learning/strategy.py",
"reinforcement_learning/agent.py",
"reinforcement_learning/environment.py",
"reinforcement_learning/replay_memory.py",
"reinforcement_learning/util.py",
"reinforcement_learning/cnnarch_logger.py"
)
);
assertTrue(Paths.get(
"./target/generated-sources-emadl/reinforcement_learning/_torcs_agent_dqn_reward_executor.so")
.toFile().exists());
assertTrue(Paths.get(
"./target/generated-sources-emadl/reinforcement_learning/torcs_agent_dqn_reward_executor.py")
.toFile().exists());
}
}
\ No newline at end of file
src/test/java/de/monticore/lang/monticar/emadl/IntegrationTest.java
View file @ 8a07e768
......@@ -26,10 +26,7 @@ import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.*;
import java.io.File;
import java.io.IOException;
......@@ -113,6 +110,7 @@ public abstract class IntegrationTest extends AbstractSymtabTest {
}
@Test
@Ignore
public void testDontRetrain2() {
// The training hash is written manually, so even the first training should be skipped
Log.getFindings().clear();
......
src/test/resources/models/Alexnet.emadl
View file @ 8a07e768
......@@ -41,7 +41,7 @@ component Alexnet{
fc(->=2) ->
FullyConnected(units=1000) ->
Softmax() ->
predictions
predictions;
}
}
\ No newline at end of file
src/test/resources/models/InstanceTest/NetworkB.emadl
View file @ 8a07e768
......@@ -18,7 +18,7 @@ component NetworkB<Z classes = 10> (Z convChannels){
Dropout() ->
FullyConnected(units=classes) ->
Softmax() ->
predictions
predictions;
}
}
\ No newline at end of file
src/test/resources/models/InvalidLayerInput.emadl
View file @ 8a07e768
......@@ -27,6 +27,6 @@ component InvalidLayerInput{
fc() ->
FullyConnected(units=1000) ->
Softmax() ->
predictions
predictions;
}
}
\ No newline at end of file
src/test/resources/models/Invariant.cnnt 0 → 100644
View file @ 8a07e768
configuration Invariant{
num_epoch:10
batch_size:5
context:cpu
optimizer:adam{
learning_rate:0.01
learning_rate_decay:0.8
step_size:1000
weight_decay:0.0001
}
}