Commit b018cdec authored by Sebastian Nickels's avatar Sebastian Nickels

Merge branch 'develop' of...

Merge branch 'develop' of git.rwth-aachen.de:monticore/EmbeddedMontiArc/generators/EMADL2CPP into develop
parents 19a6b393 71c518c8
Pipeline #172957 failed with stages
......@@ -8,7 +8,7 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.3.5-SNAPSHOT</version>
<version>0.3.6-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
......@@ -62,7 +62,6 @@
</dependency>
<!-- MontiCore Dependencies -->
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-math-opt-generator</artifactId>
......
......@@ -25,6 +25,7 @@ import de.monticore.io.paths.ModelPath;
import de.monticore.lang.embeddedmontiarc.LogConfig;
import de.monticore.lang.embeddedmontiarc.helper.ConstantPortHelper;
import de.monticore.lang.monticar.emadl._symboltable.EMADLLanguage;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathTagSchema;
import de.monticore.lang.monticar.enumlang._symboltable.EnumLangLanguage;
import de.monticore.lang.monticar.generator.cpp.converter.MathConverter;
import de.monticore.lang.monticar.generator.optimization.ThreadingOptimizer;
......@@ -58,6 +59,7 @@ public class EMADLAbstractSymtab {
TagInitTagSchema.registerTagTypes(tagging);
TagThresholdTagSchema.registerTagTypes(tagging);
TagDelayTagSchema.registerTagTypes(tagging);
DataPathTagSchema.registerTagTypes(tagging);
return tagging;
}
......
......@@ -26,42 +26,42 @@ import com.google.common.base.Splitter;
import com.google.common.io.Resources;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstantiationSymbol;
import de.monticore.lang.math._symboltable.MathStatementsSymbol;
import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.DataPathConfigParser;
import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.SerialCompositeElementSymbol;
import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnnarch.generator.DataPathConfigParser;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNTrain2Gluon;
import de.monticore.lang.monticar.cnnarch.gluongenerator.annotations.ArchitectureAdapter;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCoCoChecker;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
import de.monticore.lang.monticar.emadl._cocos.DataPathCocos;
import de.monticore.lang.monticar.emadl._cocos.EMADLCocos;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cpp.ArmadilloHelper;
import de.monticore.lang.monticar.generator.cpp.GeneratorEMAMOpt2CPP;
import de.monticore.lang.monticar.generator.cpp.SimulatorIntegrationHelper;
import de.monticore.lang.monticar.generator.cpp.TypesGeneratorCPP;
import de.monticore.lang.monticar.generator.cpp.converter.TypeConverter;
import de.monticore.lang.tagging._symboltable.TagSymbol;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.Scope;
import de.se_rwth.commons.Splitters;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import javax.xml.bind.DatatypeConverter;
import java.io.*;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.DigestInputStream;
import javax.xml.bind.DatatypeConverter;
import java.util.*;
public class EMADLGenerator {
......@@ -121,7 +121,7 @@ public class EMADLGenerator {
generateFiles(symtab, instance, symtab, pythonPath, forced);
if (doCompile) {
compile();
}
......@@ -356,6 +356,52 @@ public class EMADLGenerator {
return fileContents;
}
protected String getDataPath(TaggingResolver taggingResolver, EMAComponentSymbol component, EMAComponentInstanceSymbol instance){
List<TagSymbol> instanceTags = new LinkedList<>();
boolean isChildComponent = instance.getEnclosingComponent().isPresent();
if (isChildComponent) {
// get all instantiated components of parent
List<EMAComponentInstantiationSymbol> instantiationSymbols = (List<EMAComponentInstantiationSymbol>) instance
.getEnclosingComponent().get().getComponentType().getReferencedSymbol().getSubComponents();
// filter corresponding instantiation of instance and add tags
instantiationSymbols.stream().filter(e -> e.getName().equals(instance.getName())).findFirst()
.ifPresent(symbol -> instanceTags.addAll(taggingResolver.getTags(symbol, DataPathSymbol.KIND)));
}
// instance tags have priority
List<TagSymbol> tags = !instanceTags.isEmpty() ? instanceTags
: (List<TagSymbol>) taggingResolver.getTags(component, DataPathSymbol.KIND);
String dataPath;
if (!tags.isEmpty()) {
DataPathSymbol dataPathSymbol = (DataPathSymbol) tags.get(0);
DataPathCocos.check(dataPathSymbol);
dataPath = dataPathSymbol.getPath();
// TODO: Replace warinings with errors, until then use this method
stopGeneratorIfWarning();
Log.warn("Tagging info for symbol was found, ignoring data_paths.txt: " + dataPath);
}
else {
Path dataPathDefinition = Paths.get(getModelsPath(), "data_paths.txt");
if (dataPathDefinition.toFile().exists()) {
DataPathConfigParser newParserConfig = new DataPathConfigParser(getModelsPath() + "data_paths.txt");
dataPath = newParserConfig.getDataPath(component.getFullName());
} else {
Log.warn("No data path definition found in " + dataPathDefinition + " found: "
+ "Set data path to default ./data path");
dataPath = "data";
}
}
return dataPath;
}
protected void generateComponent(List<FileContent> fileContents,
Set<EMAComponentInstanceSymbol> allInstances,
TaggingResolver taggingResolver,
......@@ -376,19 +422,7 @@ public class EMADLGenerator {
if (architecture.isPresent()){
cnnArchGenerator.check(architecture.get());
String dPath = null;
Path dataPathDefinition = Paths.get(getModelsPath(), "data_paths.txt");
if (dataPathDefinition.toFile().exists()) {
DataPathConfigParser newParserConfig = new DataPathConfigParser(getModelsPath() + "data_paths.txt");
dPath = newParserConfig.getDataPath(EMAComponentSymbol.getFullName());
} else {
Log.warn("No data path definition found in " + dataPathDefinition + " found: "
+ "Set data path to default ./data path");
dPath = "data";
}
/*String dPath = DataPathConfigParser.getDataPath(getModelsPath() + "data_paths.txt", componentSymbol.getFullName());*/
String dPath = getDataPath(taggingResolver, EMAComponentSymbol, componentInstanceSymbol);
architecture.get().setDataPath(dPath);
architecture.get().setComponentName(EMAComponentSymbol.getFullName());
generateCNN(fileContents, taggingResolver, componentInstanceSymbol, architecture.get());
......@@ -587,4 +621,16 @@ public class EMADLGenerator {
return null;
}
}
private void stopGeneratorIfWarning() {
for (int i = 0; i < Log.getFindings().size(); i++) {
if (Log.getFindings().get(i).toString().matches("Filepath '(.)*' does not exist!")) {
throw new RuntimeException(Log.getFindings().get(i).toString());
} else if (Log.getFindings().get(i).toString()
.equals("DatapathType is incorrect, must be of Type: HDF5 or LMDB")) {
throw new RuntimeException(Log.getFindings().get(i).toString());
}
}
}
}
......@@ -27,21 +27,15 @@ import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class GenerationTest extends AbstractSymtabTest {
@Before
......@@ -54,7 +48,8 @@ public class GenerationTest extends AbstractSymtabTest {
@Test
public void testCifar10Generation() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET", "-f", "n", "-c", "n"};
String[] args = { "-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET", "-f",
"n", "-c", "n" };
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
......@@ -286,4 +281,58 @@ public class GenerationTest extends AbstractSymtabTest {
)
);
}
@Test
public void testAlexNetTagging() {
Log.getFindings().clear();
String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.Alexnet", "-b", "MXNET", "-f", "n", "-c",
"n" };
EMADLGeneratorCli.main(args);
assertEquals(Log.getFindings().size(), 1);
assertEquals(Log.getFindings().get(0).toString(),
"Tagging info for symbol was found, ignoring data_paths.txt: src/test/resources/models");
assertTrue(Log.getErrorCount() == 0);
}
@Test
public void testAlexNetTaggingForInstances() {
Log.getFindings().clear();
String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.Parent", "-b", "MXNET", "-f", "n", "-c",
"n" };
EMADLGeneratorCli.main(args);
}
@Test(expected = RuntimeException.class)
public void testInvalidPathCoCos() {
Log.getFindings().clear();
String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.AlexnetInvalid", "-b", "MXNET", "-f", "n",
"-c", "n" };
EMADLGeneratorCli.main(args);
}
@Test(expected = RuntimeException.class)
public void testInvalidPathCoCosInstances() {
Log.getFindings().clear();
String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.ParentInvalidPath", "-b", "MXNET", "-f",
"n", "-c", "n" };
EMADLGeneratorCli.main(args);
}
@Test(expected = RuntimeException.class)
public void testInvalidTypeCocos() {
Log.getFindings().clear();
String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.AlexnetInvalidType", "-b", "MXNET", "-f",
"n", "-c", "n" };
EMADLGeneratorCli.main(args);
}
@Test(expected = RuntimeException.class)
public void testInvalidTypeCocosInstances() {
Log.getFindings().clear();
String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.ParentInvalidType", "-b", "MXNET", "-f",
"n", "-c", "n" };
EMADLGeneratorCli.main(args);
}
}
package tagging;
conforms to dltag.DataPathTagSchema;
tags Alexnet {
tag Alexnet with DataPath = {path = src/test/resources/models, type = LMDB};
tag AlexnetInvalid with DataPath = {path = test/resources/models, type = random};
tag AlexnetInvalidType with DataPath = {path = src/test/resources/models, type = LMBD};
tag Parent.a1 with DataPath = {path = src/test/resources/models, type = LMDB};
tag Parent.a2 with DataPath = {path = src/test/resources/models, type = LMDB};
tag ParentInvalidType.a1 with DataPath = {path = src/test/resources/models, type = LMDB};
tag ParentInvalidType.a2 with DataPath = {path = src/test/resources/models, type = LMDB_1};
tag ParentInvalidPath.a1 with DataPath = {path = /NO_VALID_PATH/, type = LMDB};
tag ParentInvalidPath.a2 with DataPath = {path = src/test/resources/models, type = LMDB};
}
configuration Alexnet{
num_epoch : 100
batch_size : 500
optimizer : adam{
learning_rate : 0.001
}
}
package tagging;
component Alexnet{
ports in Z(0:255)^{3, 224, 224} image,
out Q(0:1)^{1000} predictions;
implementation CNN {
def split1(i){
[i] ->
Convolution(kernel=(5,5), channels=128) ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def split2(i){
[i] ->
Convolution(kernel=(3,3), channels=192) ->
Relu() ->
Convolution(kernel=(3,3), channels=128) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def fc(){
FullyConnected(units=4096) ->
Relu() ->
Dropout()
}
image ->
Convolution(kernel=(11,11), channels=96, stride=(4,4), padding="no_loss") ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu() ->
Split(n=2) ->
split1(i=[0|1]) ->
Concatenate() ->
Convolution(kernel=(3,3), channels=384) ->
Relu() ->
Split(n=2) ->
split2(i=[0|1]) ->
Concatenate() ->
fc(->=2) ->
FullyConnected(units=1000) ->
Softmax() ->
predictions;
}
}
\ No newline at end of file
configuration AlexnetInvalid{
num_epoch : 100
batch_size : 500
optimizer : adam{
learning_rate : 0.001
}
}
package tagging;
component AlexnetInvalid{
ports in Z(0:255)^{3, 224, 224} image,
out Q(0:1)^{1000} predictions;
implementation CNN {
def split1(i){
[i] ->
Convolution(kernel=(5,5), channels=128) ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def split2(i){
[i] ->
Convolution(kernel=(3,3), channels=192) ->
Relu() ->
Convolution(kernel=(3,3), channels=128) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def fc(){
FullyConnected(units=4096) ->
Relu() ->
Dropout()
}
image ->
Convolution(kernel=(11,11), channels=96, stride=(4,4), padding="no_loss") ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu() ->
Split(n=2) ->
split1(i=[0|1]) ->
Concatenate() ->
Convolution(kernel=(3,3), channels=384) ->
Relu() ->
Split(n=2) ->
split2(i=[0|1]) ->
Concatenate() ->
fc(->=2) ->
FullyConnected(units=1000) ->
Softmax() ->
predictions;
}
}
\ No newline at end of file
configuration AlexnetInvalidType{
num_epoch : 100
batch_size : 500
optimizer : adam{
learning_rate : 0.001
}
}
package tagging;
component AlexnetInvalidType{
ports in Z(0:255)^{3, 224, 224} image,
out Q(0:1)^{1000} predictions;
implementation CNN {
def split1(i){
[i] ->
Convolution(kernel=(5,5), channels=128) ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def split2(i){
[i] ->
Convolution(kernel=(3,3), channels=192) ->
Relu() ->
Convolution(kernel=(3,3), channels=128) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def fc(){
FullyConnected(units=4096) ->
Relu() ->
Dropout()
}
image ->
Convolution(kernel=(11,11), channels=96, stride=(4,4), padding="no_loss") ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu() ->
Split(n=2) ->
split1(i=[0|1]) ->
Concatenate() ->
Convolution(kernel=(3,3), channels=384) ->
Relu() ->
Split(n=2) ->
split2(i=[0|1]) ->
Concatenate() ->
fc(->=2) ->
FullyConnected(units=1000) ->
Softmax() ->
predictions;
}
}
\ No newline at end of file
package tagging;
component Parent {
instance tagging.Alexnet a1;
}
\ No newline at end of file
package tagging;
component ParentInvalidPath{
instance Alexnet a1;
instance Alexnet a2;
}
\ No newline at end of file
package tagging;
component ParentInvalidType{
instance Alexnet a1;
instance Alexnet a2;
}
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment