Merge branch 'develop' of...

Merge branch 'develop' of git.rwth-aachen.de:monticore/EmbeddedMontiArc/generators/EMADL2CPP into develop

pom.xml

@@ -8,7 +8,7 @@
 
   <groupId>de.monticore.lang.monticar</groupId>
   <artifactId>embedded-montiarc-emadl-generator</artifactId>
-  <version>0.3.5-SNAPSHOT</version>
+  <version>0.3.6-SNAPSHOT</version>
 
   <!-- == PROJECT DEPENDENCIES ============================================= -->
 
@@ -62,7 +62,6 @@
         </dependency>
 
         <!-- MontiCore Dependencies -->
-
         <dependency>
             <groupId>de.monticore.lang.monticar</groupId>
             <artifactId>embedded-montiarc-math-opt-generator</artifactId>

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLAbstractSymtab.java

@@ -25,6 +25,7 @@ import de.monticore.io.paths.ModelPath;
 import de.monticore.lang.embeddedmontiarc.LogConfig;
 import de.monticore.lang.embeddedmontiarc.helper.ConstantPortHelper;
 import de.monticore.lang.monticar.emadl._symboltable.EMADLLanguage;
+import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathTagSchema;
 import de.monticore.lang.monticar.enumlang._symboltable.EnumLangLanguage;
 import de.monticore.lang.monticar.generator.cpp.converter.MathConverter;
 import de.monticore.lang.monticar.generator.optimization.ThreadingOptimizer;
@@ -58,6 +59,7 @@ public class EMADLAbstractSymtab {
         TagInitTagSchema.registerTagTypes(tagging);
         TagThresholdTagSchema.registerTagTypes(tagging);
         TagDelayTagSchema.registerTagTypes(tagging);
+        DataPathTagSchema.registerTagTypes(tagging);
         return tagging;
     }
 

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGenerator.java

@@ -26,42 +26,42 @@ import com.google.common.base.Splitter;
 import com.google.common.io.Resources;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstantiationSymbol;
 import de.monticore.lang.math._symboltable.MathStatementsSymbol;
-import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
-import de.monticore.lang.monticar.cnnarch.generator.DataPathConfigParser;
-import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
 import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
 import de.monticore.lang.monticar.cnnarch._symboltable.SerialCompositeElementSymbol;
+import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
+import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
+import de.monticore.lang.monticar.cnnarch.generator.DataPathConfigParser;
 import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNTrain2Gluon;
 import de.monticore.lang.monticar.cnnarch.gluongenerator.annotations.ArchitectureAdapter;
-import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCoCoChecker;
 import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
 import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
+import de.monticore.lang.monticar.emadl._cocos.DataPathCocos;
 import de.monticore.lang.monticar.emadl._cocos.EMADLCocos;
+import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
 import de.monticore.lang.monticar.generator.FileContent;
 import de.monticore.lang.monticar.generator.cpp.ArmadilloHelper;
 import de.monticore.lang.monticar.generator.cpp.GeneratorEMAMOpt2CPP;
 import de.monticore.lang.monticar.generator.cpp.SimulatorIntegrationHelper;
 import de.monticore.lang.monticar.generator.cpp.TypesGeneratorCPP;
 import de.monticore.lang.monticar.generator.cpp.converter.TypeConverter;
+import de.monticore.lang.tagging._symboltable.TagSymbol;
 import de.monticore.lang.tagging._symboltable.TaggingResolver;
 import de.monticore.symboltable.Scope;
 import de.se_rwth.commons.Splitters;
 import de.se_rwth.commons.logging.Log;
 import freemarker.template.TemplateException;
 
+import javax.xml.bind.DatatypeConverter;
 import java.io.*;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.*;
-
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.security.DigestInputStream;
-
-import javax.xml.bind.DatatypeConverter;
+import java.util.*;
 
 public class EMADLGenerator {
 
@@ -121,7 +121,7 @@ public class EMADLGenerator {
 
 
         generateFiles(symtab, instance, symtab, pythonPath, forced);
-        
+
         if (doCompile) {
             compile();
         }
@@ -356,6 +356,52 @@ public class EMADLGenerator {
         return fileContents;
     }
 
+    protected String getDataPath(TaggingResolver taggingResolver, EMAComponentSymbol component, EMAComponentInstanceSymbol instance){
+        List<TagSymbol> instanceTags = new LinkedList<>();
+
+        boolean isChildComponent = instance.getEnclosingComponent().isPresent();
+
+        if (isChildComponent) {
+            // get all instantiated components of parent
+            List<EMAComponentInstantiationSymbol> instantiationSymbols = (List<EMAComponentInstantiationSymbol>) instance
+                    .getEnclosingComponent().get().getComponentType().getReferencedSymbol().getSubComponents();
+
+            // filter corresponding instantiation of instance and add tags
+            instantiationSymbols.stream().filter(e -> e.getName().equals(instance.getName())).findFirst()
+                    .ifPresent(symbol -> instanceTags.addAll(taggingResolver.getTags(symbol, DataPathSymbol.KIND)));
+        }
+
+        // instance tags have priority
+        List<TagSymbol> tags = !instanceTags.isEmpty() ? instanceTags
+                : (List<TagSymbol>) taggingResolver.getTags(component, DataPathSymbol.KIND);
+
+        String dataPath;
+
+        if (!tags.isEmpty()) {
+            DataPathSymbol dataPathSymbol = (DataPathSymbol) tags.get(0);
+            DataPathCocos.check(dataPathSymbol);
+
+            dataPath = dataPathSymbol.getPath();
+
+            // TODO: Replace warinings with errors, until then use this method
+            stopGeneratorIfWarning();
+            Log.warn("Tagging info for symbol was found, ignoring data_paths.txt: " + dataPath);
+        }
+        else {
+            Path dataPathDefinition = Paths.get(getModelsPath(), "data_paths.txt");
+            if (dataPathDefinition.toFile().exists()) {
+                DataPathConfigParser newParserConfig = new DataPathConfigParser(getModelsPath() + "data_paths.txt");
+                dataPath = newParserConfig.getDataPath(component.getFullName());
+            } else {
+                Log.warn("No data path definition found in " + dataPathDefinition + " found: "
+                        + "Set data path to default ./data path");
+                dataPath = "data";
+            }
+        }
+
+        return dataPath;
+    }
+
     protected void generateComponent(List<FileContent> fileContents,
                                      Set<EMAComponentInstanceSymbol> allInstances,
                                      TaggingResolver taggingResolver,
@@ -376,19 +422,7 @@ public class EMADLGenerator {
 
         if (architecture.isPresent()){
             cnnArchGenerator.check(architecture.get());
-
-            String dPath = null;
-            Path dataPathDefinition = Paths.get(getModelsPath(), "data_paths.txt");
-            if (dataPathDefinition.toFile().exists()) {
-                DataPathConfigParser newParserConfig = new DataPathConfigParser(getModelsPath() + "data_paths.txt");
-                dPath = newParserConfig.getDataPath(EMAComponentSymbol.getFullName());
-            } else {
-                Log.warn("No data path definition found in " + dataPathDefinition + " found: "
-                        + "Set data path to default ./data path");
-                dPath = "data";
-            }
-
-            /*String dPath = DataPathConfigParser.getDataPath(getModelsPath() + "data_paths.txt", componentSymbol.getFullName());*/
+            String dPath = getDataPath(taggingResolver, EMAComponentSymbol, componentInstanceSymbol);
             architecture.get().setDataPath(dPath);
             architecture.get().setComponentName(EMAComponentSymbol.getFullName());
             generateCNN(fileContents, taggingResolver, componentInstanceSymbol, architecture.get());
@@ -587,4 +621,16 @@ public class EMADLGenerator {
             return null;
         }
     }
+
+    private void stopGeneratorIfWarning() {
+        for (int i = 0; i < Log.getFindings().size(); i++) {
+            if (Log.getFindings().get(i).toString().matches("Filepath '(.)*' does not exist!")) {
+                throw new RuntimeException(Log.getFindings().get(i).toString());
+            } else if (Log.getFindings().get(i).toString()
+                    .equals("DatapathType is incorrect, must be of Type: HDF5 or LMDB")) {
+                throw new RuntimeException(Log.getFindings().get(i).toString());
+            }
+        }
+    }
+
 }

src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java

@@ -27,21 +27,15 @@ import de.se_rwth.commons.logging.Finding;
 import de.se_rwth.commons.logging.Log;
 import freemarker.template.TemplateException;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.List;
 import java.util.stream.Collectors;
 
 import static junit.framework.TestCase.assertEquals;
 import static junit.framework.TestCase.assertTrue;
-import static org.junit.Assert.assertFalse;
 
 public class GenerationTest extends AbstractSymtabTest {
     @Before
@@ -54,7 +48,8 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testCifar10Generation() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET", "-f", "n", "-c", "n"};
+        String[] args = { "-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET", "-f",
+                "n", "-c", "n" };
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
 
@@ -286,4 +281,58 @@ public class GenerationTest extends AbstractSymtabTest {
                 )
         );
     }
+
+    @Test
+    public void testAlexNetTagging() {
+        Log.getFindings().clear();
+        String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.Alexnet", "-b", "MXNET", "-f", "n", "-c",
+                "n" };
+        EMADLGeneratorCli.main(args);
+        assertEquals(Log.getFindings().size(), 1);
+        assertEquals(Log.getFindings().get(0).toString(),
+                "Tagging info for symbol was found, ignoring data_paths.txt: src/test/resources/models");
+        assertTrue(Log.getErrorCount() == 0);
+    }
+
+    @Test
+    public void testAlexNetTaggingForInstances() {
+        Log.getFindings().clear();
+        String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.Parent", "-b", "MXNET", "-f", "n", "-c",
+                "n" };
+        EMADLGeneratorCli.main(args);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testInvalidPathCoCos() {
+        Log.getFindings().clear();
+        String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.AlexnetInvalid", "-b", "MXNET", "-f", "n",
+                "-c", "n" };
+        EMADLGeneratorCli.main(args);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testInvalidPathCoCosInstances() {
+        Log.getFindings().clear();
+        String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.ParentInvalidPath", "-b", "MXNET", "-f",
+                "n", "-c", "n" };
+        EMADLGeneratorCli.main(args);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testInvalidTypeCocos() {
+
+        Log.getFindings().clear();
+        String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.AlexnetInvalidType", "-b", "MXNET", "-f",
+                "n", "-c", "n" };
+        EMADLGeneratorCli.main(args);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testInvalidTypeCocosInstances() {
+
+        Log.getFindings().clear();
+        String[] args = { "-m", "src/test/resources/models/", "-r", "tagging.ParentInvalidType", "-b", "MXNET", "-f",
+                "n", "-c", "n" };
+        EMADLGeneratorCli.main(args);
+    }
 }

src/test/resources/models/tagging/AlexNet.tag

+package tagging;
+conforms to dltag.DataPathTagSchema;
+
+tags Alexnet {
+tag Alexnet with DataPath = {path = src/test/resources/models, type = LMDB};
+tag AlexnetInvalid with DataPath = {path = test/resources/models, type = random};
+tag AlexnetInvalidType with DataPath = {path = src/test/resources/models, type = LMBD};
+tag Parent.a1 with DataPath = {path = src/test/resources/models, type = LMDB};
+tag Parent.a2 with DataPath = {path = src/test/resources/models, type = LMDB};
+tag ParentInvalidType.a1 with DataPath = {path = src/test/resources/models, type = LMDB};
+tag ParentInvalidType.a2 with DataPath = {path = src/test/resources/models, type = LMDB_1};
+tag ParentInvalidPath.a1 with DataPath = {path = /NO_VALID_PATH/, type = LMDB};
+tag ParentInvalidPath.a2 with DataPath = {path = src/test/resources/models, type = LMDB};
+}
+
+
+

src/test/resources/models/tagging/Alexnet.cnnt

+configuration Alexnet{
+     num_epoch : 100
+     batch_size : 500
+     optimizer : adam{
+         learning_rate : 0.001
+     }
+}

src/test/resources/models/tagging/Alexnet.emadl

+package tagging;
+
+component Alexnet{
+    ports in Z(0:255)^{3, 224, 224} image,
+         out Q(0:1)^{1000} predictions;
+
+    implementation CNN {
+
+        def split1(i){
+            [i] ->
+            Convolution(kernel=(5,5), channels=128) ->
+            Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def split2(i){
+            [i] ->
+            Convolution(kernel=(3,3), channels=192) ->
+            Relu() ->
+            Convolution(kernel=(3,3), channels=128) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def fc(){
+            FullyConnected(units=4096) ->
+            Relu() ->
+            Dropout()
+        }
+
+        image ->
+        Convolution(kernel=(11,11), channels=96, stride=(4,4), padding="no_loss") ->
+        Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+        Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+        Relu() ->
+        Split(n=2) ->
+        split1(i=[0|1]) ->
+        Concatenate() ->
+        Convolution(kernel=(3,3), channels=384) ->
+        Relu() ->
+        Split(n=2) ->
+        split2(i=[0|1]) ->
+        Concatenate() ->
+        fc(->=2) ->
+        FullyConnected(units=1000) ->
+        Softmax() ->
+        predictions;
+
+    }
+}
\ No newline at end of file

src/test/resources/models/tagging/AlexnetInvalid.cnnt

+configuration AlexnetInvalid{
+     num_epoch : 100
+     batch_size : 500
+     optimizer : adam{
+         learning_rate : 0.001
+     }
+}

src/test/resources/models/tagging/AlexnetInvalid.emadl

+package tagging;
+
+component AlexnetInvalid{
+    ports in Z(0:255)^{3, 224, 224} image,
+         out Q(0:1)^{1000} predictions;
+
+    implementation CNN {
+
+        def split1(i){
+            [i] ->
+            Convolution(kernel=(5,5), channels=128) ->
+            Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def split2(i){
+            [i] ->
+            Convolution(kernel=(3,3), channels=192) ->
+            Relu() ->
+            Convolution(kernel=(3,3), channels=128) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def fc(){
+            FullyConnected(units=4096) ->
+            Relu() ->
+            Dropout()
+        }
+
+        image ->
+        Convolution(kernel=(11,11), channels=96, stride=(4,4), padding="no_loss") ->
+        Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+        Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+        Relu() ->
+        Split(n=2) ->
+        split1(i=[0|1]) ->
+        Concatenate() ->
+        Convolution(kernel=(3,3), channels=384) ->
+        Relu() ->
+        Split(n=2) ->
+        split2(i=[0|1]) ->
+        Concatenate() ->
+        fc(->=2) ->
+        FullyConnected(units=1000) ->
+        Softmax() ->
+        predictions;
+
+    }
+}
\ No newline at end of file

src/test/resources/models/tagging/AlexnetInvalidType.cnnt

+configuration AlexnetInvalidType{
+     num_epoch : 100
+     batch_size : 500
+     optimizer : adam{
+         learning_rate : 0.001
+     }
+}

src/test/resources/models/tagging/AlexnetInvalidType.emadl

+package tagging;
+
+component AlexnetInvalidType{
+    ports in Z(0:255)^{3, 224, 224} image,
+         out Q(0:1)^{1000} predictions;
+
+    implementation CNN {
+
+        def split1(i){
+            [i] ->
+            Convolution(kernel=(5,5), channels=128) ->
+            Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def split2(i){
+            [i] ->
+            Convolution(kernel=(3,3), channels=192) ->
+            Relu() ->
+            Convolution(kernel=(3,3), channels=128) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def fc(){
+            FullyConnected(units=4096) ->
+            Relu() ->
+            Dropout()
+        }
+
+        image ->
+        Convolution(kernel=(11,11), channels=96, stride=(4,4), padding="no_loss") ->
+        Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+        Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+        Relu() ->
+        Split(n=2) ->
+        split1(i=[0|1]) ->
+        Concatenate() ->
+        Convolution(kernel=(3,3), channels=384) ->
+        Relu() ->
+        Split(n=2) ->
+        split2(i=[0|1]) ->
+        Concatenate() ->
+        fc(->=2) ->
+        FullyConnected(units=1000) ->
+        Softmax() ->
+        predictions;
+
+    }
+}
\ No newline at end of file

src/test/resources/models/tagging/Parent.emadl

+package tagging;
+
+component Parent {
+    instance tagging.Alexnet a1;
+}
\ No newline at end of file

src/test/resources/models/tagging/ParentInvalidPath.emadl

+package tagging;
+
+component ParentInvalidPath{
+    instance Alexnet a1;
+    instance Alexnet a2;
+}
\ No newline at end of file

src/test/resources/models/tagging/ParentInvalidType.emadl

+package tagging;
+
+component ParentInvalidType{
+    instance Alexnet a1;
+    instance Alexnet a2;
+}
\ No newline at end of file