Merge branch 'feature/path_tagging' into 'master'

Added Datapath-Tagging for Deep Learning Components

See merge request !15

pom.xml

@@ -30,7 +30,7 @@
 
     <groupId>de.monticore.lang.monticar</groupId>
     <artifactId>embedded-montiarc-deeplearning</artifactId>
-    <version>0.2.8-SNAPSHOT</version>
+    <version>0.2.9-SNAPSHOT</version>
 
     <!-- == PROJECT DEPENDENCIES ============================================= -->
 

src/main/java/de/monticore/lang/monticar/emadl/_cocos/DataPathCocos.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl._cocos;
+
+import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
+
+import de.se_rwth.commons.logging.Log;
+import java.io.File;
+
+public class DataPathCocos {
+
+    public static void check(DataPathSymbol dataPathSymbol) {
+        File dataPath = new File(dataPathSymbol.getPath());
+
+        if (!dataPath.exists()) {
+            Log.warn(String.format("Filepath '%s' does not exist!", dataPath.getAbsolutePath()));
+           //Log.error(String.format("Filepath '%s' does not exist!", dataPath.getAbsolutePath()));
+           // TODO should be error, but test exits abruptly if set to error
+        }
+
+        if (!dataPathSymbol.getType().equals("HDF5") && !dataPathSymbol.getType().equals("LMDB")) {
+            Log.warn("DatapathType is incorrect, must be of Type: HDF5 or LMDB");
+            //Log.error("DatapathType is incorrect, must be of Type: HDF5 or LMDB");
+            // TODO should be error, but test exits abruptly if set to error
+        }
+    }
+}

src/main/java/de/monticore/lang/monticar/emadl/tagging/dltag/DataPathSymbol.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl.tagging.dltag;
+
+import de.monticore.lang.tagging._symboltable.TagKind;
+import de.monticore.lang.tagging._symboltable.TagSymbol;
+
+public class DataPathSymbol extends TagSymbol {
+    public static final DataPathKind KIND  = DataPathKind.INSTANCE;
+
+    public DataPathSymbol() {
+        super(KIND, ".");
+    }
+
+    public DataPathSymbol(String path, String type) {
+        this(KIND, path, type);
+    }
+
+    public DataPathSymbol(DataPathKind kind, String path, String type) {
+        super(kind, path, type);
+    }
+
+    public String getPath() {
+      return getValue(0);
+    }
+
+    public String getType() {
+      return getValue(1);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString();
+    }
+
+    public static class DataPathKind extends TagKind {
+        public static final DataPathKind INSTANCE = new DataPathKind();
+
+        protected DataPathKind() {
+        }
+    }
+}

src/main/java/de/monticore/lang/monticar/emadl/tagging/dltag/DataPathSymbolCreator.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl.tagging.dltag;
+
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
+import de.monticore.lang.embeddedmontiarcdynamic.embeddedmontiarcdynamic._symboltable.instanceStructure.EMADynamicComponentInstantiationSymbol;
+import de.monticore.lang.tagging._ast.ASTNameScope;
+import de.monticore.lang.tagging._ast.ASTTag;
+import de.monticore.lang.tagging._ast.ASTTagBody;
+import de.monticore.lang.tagging._ast.ASTTaggingUnit;
+import de.monticore.lang.tagging._symboltable.TagSymbolCreator;
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+import de.monticore.symboltable.SymbolKind;
+import de.monticore.types.types._ast.ASTQualifiedName;
+import de.se_rwth.commons.Joiners;
+import de.se_rwth.commons.logging.Log;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DataPathSymbolCreator implements TagSymbolCreator {
+
+    private final String regexExpression =  "\\s*\\{\\s*path\\s*=\\s*(.*)\\s*,\\s*type\\s*=\\s*(.*)\\s*\\}\\s*";
+    private final Pattern pattern = Pattern.compile(regexExpression, Pattern.MULTILINE);
+
+    @Override
+    public void create(ASTTaggingUnit unit, TaggingResolver tagging) {
+        boolean hasDataPathTag =
+            unit.getQualifiedNameList().stream()
+                .map(ASTQualifiedName::toString)
+                .anyMatch(n -> n.endsWith("DataPathTagSchema"));
+
+        if (!hasDataPathTag) {
+            return;
+        }
+
+        final String packageName = Joiners.DOT.join(unit.getPackageList());
+        final ASTTagBody tagBody = unit.getTagBody();
+        final String root =
+            (tagBody.getTargetModelOpt().isPresent()) ?
+                Joiners.DOT.join(packageName, tagBody.getTargetModelOpt().get()
+                    .getQualifiedNameString()) :
+                packageName;
+
+
+        for (ASTTag tag : unit.getTagBody().getTagList()) {
+            addTag(tag, tagging, root, EMADynamicComponentInstantiationSymbol.KIND);
+            addTag(tag, tagging, root, EMAComponentSymbol.KIND);
+        }
+    }
+
+    private void addTag(ASTTag tag, TaggingResolver tagging, String root, SymbolKind kind) {
+        tag.getTagElementList().stream()
+            .filter(tagElement -> tagElement.getName().equals("DataPath"))
+            .forEachOrdered(tagElement -> {
+                Matcher matcher = matchRegexPattern(tagElement.getTagValue());
+                tag.getScopeList().stream()
+                    .filter(scope -> scope.getScopeKind().equals("NameScope"))
+                    .map(scope -> (ASTNameScope) scope)
+                    .map(scope ->
+                            tagging.resolve(dotJoin(root, scope.getQualifiedNameString()), kind)
+                    )
+                    .filter(Optional::isPresent)
+                    .map(Optional::get)
+                    .forEachOrdered(scope ->
+                            tagging.addTag(scope, new DataPathSymbol(matcher.group(1), matcher.group(2)))
+                    );
+            });
+    }
+
+    private Matcher matchRegexPattern(String regex) {
+        Matcher matcher = pattern.matcher(regex);
+        if (matcher.matches()) {
+            return matcher;
+        }
+        else {
+            Log.error(
+                String.format(
+                        "'%s' does not match the specified regex pattern '%s'",
+                        regex, "{path = {dataPath}, type = {dataType}}"
+                )
+            );
+            return null;
+        }
+
+    }
+
+    private String dotJoin(String root, String name) {
+        if (StringUtils.isEmpty(root)) {
+            return name;
+        }
+        else {
+            return Joiners.DOT.join(root, name);
+        }
+    }
+
+}

src/main/java/de/monticore/lang/monticar/emadl/tagging/dltag/DataPathTagSchema.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl.tagging.dltag;
+
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+import de.monticore.symboltable.resolving.CommonResolvingFilter;
+
+public class DataPathTagSchema {
+
+    protected static DataPathTagSchema instance = null;
+
+    protected DataPathTagSchema() {
+
+    }
+
+    protected static DataPathTagSchema getInstance() {
+        if (instance == null) {
+            instance = new DataPathTagSchema();
+        }
+        return instance;
+    }
+
+    protected void doRegisterTagTypes(TaggingResolver tagging) {
+        tagging.addTagSymbolCreator(new DataPathSymbolCreator());
+        tagging.addTagSymbolResolvingFilter(CommonResolvingFilter.create(DataPathSymbol.KIND));
+    }
+
+    public static void registerTagTypes(TaggingResolver tagging) {
+            getInstance().doRegisterTagTypes(tagging);
+    }
+
+}

src/main/tagschemas/dltag/DataPathTagSchema.tagschema

+package dltag;
+
+tagschema DeepLearningFilePathSchema {
+	tagtype DataPath for EMAComponentSymbol is { path = ${path:String}, type = ${type:String} };
+}
\ No newline at end of file

src/test/java/de/monticore/lang/monticar/emadl/AbstractTaggingResolverTest.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl;
+
+import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathTagSchema;
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+import de.monticore.symboltable.Scope;
+
+import java.util.Arrays;
+
+public class AbstractTaggingResolverTest extends AbstractSymtabTest {
+
+    protected static TaggingResolver createSymTabandTaggingResolver(String... modelPath) {
+        Scope scope = createSymTab(modelPath);
+        TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));
+        DataPathTagSchema.registerTagTypes(tagging);
+
+        return tagging;
+    }
+
+}

src/test/java/de/monticore/lang/monticar/emadl/cocos/AbstractCoCoTest.java

@@ -85,6 +85,7 @@ public class AbstractCoCoTest extends AbstractSymtabTest {
         // check whether all the expected errors are present when using all cocos
         Log.getFindings().clear();
         EMADLCocos.checkAll(getComponentInstance(modelPath, model));
+        EMADLCocos.checkAll(getComponentInstance(modelPath, model));
         expectedErrors.checkExpectedPresent(Log.getFindings(), "Got no findings when checking all "
                 + "cocos. Did you forget to add the new coco to MontiArcCocos?");
     }

src/test/java/de/monticore/lang/monticar/emadl/cocos/AllCoCoTest.java

@@ -65,4 +65,4 @@ public class AllCoCoTest extends AbstractCoCoTest {
         checkValid("models", "MinimizePortsTest");
     }
 
-}
\ No newline at end of file
+}

src/test/java/de/monticore/lang/monticar/emadl/cocos/TaggingCoCoTest.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl.cocos;
+
+
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
+import de.monticore.lang.embeddedmontiarcdynamic.embeddedmontiarcdynamic._symboltable.instanceStructure.EMADynamicComponentInstantiationSymbol;
+import de.monticore.lang.tagging._symboltable.TagSymbol;
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
+import de.monticore.lang.monticar.cnnarch.helper.ErrorCodes;
+import de.monticore.lang.monticar.emadl._cocos.DataPathCocos;
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+
+import de.se_rwth.commons.logging.Log;
+import org.junit.Before;
+import org.junit.Test;
+import java.util.Collection;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import de.monticore.lang.monticar.emadl.AbstractTaggingResolverTest;
+
+import static de.monticore.lang.monticar.emadl.ParserTest.ENABLE_FAIL_QUICK;
+
+public class TaggingCoCoTest extends AbstractTaggingResolverTest {
+    @Before
+    public void setUp() {
+        // ensure an empty log
+        Log.getFindings().clear();
+        Log.enableFailQuick(ENABLE_FAIL_QUICK);
+    }
+
+    @Test
+    public void testCoCosWithInvalidType() {
+        TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
+        EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.Alexnet", EMAComponentSymbol.KIND)
+            .orElse(null);
+        assertNotNull(symbol);
+
+        Collection<TagSymbol> tags = tagging.getTags(symbol, DataPathSymbol.KIND);
+        assertEquals(1, tags.size());
+        DataPathSymbol tag = (DataPathSymbol) tags.iterator().next();
+        checkValid(tag);
+
+        assertFalse(Log.getFindings().isEmpty());
+    }
+
+    @Test
+    public void testCoCosWithValidType() {
+        TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
+        EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.CorrectTypeNet", EMAComponentSymbol.KIND)
+            .orElse(null);
+        assertNotNull(symbol);
+
+        Collection<TagSymbol> tags = tagging.getTags(symbol, DataPathSymbol.KIND);
+        assertEquals(1, tags.size());
+        DataPathSymbol tag = (DataPathSymbol) tags.iterator().next();
+        checkValid(tag);
+
+        assertTrue(Log.getFindings().isEmpty());
+    }
+
+    @Test
+    public void testCoCosForInstancesWithValidType() {
+        TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources/");
+        EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.CorrectTypeInstance", EMAComponentSymbol.KIND)
+            .orElse(null);
+        assertNotNull(symbol);
+
+        Collection<TagSymbol> tagsNet1 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("net1").iterator().next(), DataPathSymbol.KIND);
+        assertEquals(1, tagsNet1.size());
+        checkValid((DataPathSymbol) tagsNet1.iterator().next());
+
+        Collection<TagSymbol> tagsNet2 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("net2").iterator().next(), DataPathSymbol.KIND);
+        assertEquals(1, tagsNet2.size());
+        checkValid((DataPathSymbol) tagsNet2.iterator().next());
+
+        assertTrue(Log.getFindings().isEmpty());
+    }
+
+    protected static void checkValid(DataPathSymbol dataPathSymbol) {
+        Log.getFindings().clear();
+        DataPathCocos.check(dataPathSymbol);
+    }
+}

src/test/java/de/monticore/lang/monticar/emadl/tagging/dltag/TaggingTest.java

+/**
+ *
+ *  ******************************************************************************
+ *  MontiCAR Modeling Family, www.se-rwth.de
+ *  Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
+ *  All rights reserved.
+ *
+ *  This project is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 3.0 of the License, or (at your option) any later version.
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this project. If not, see <http://www.gnu.org/licenses/>.
+ * *******************************************************************************
+ */
+package de.monticore.lang.monticar.emadl.tagging.dltag;
+
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
+
+import de.monticore.lang.monticar.emadl.AbstractTaggingResolverTest;
+import de.monticore.lang.tagging._symboltable.TagSymbol;
+import de.monticore.lang.tagging._symboltable.TaggingResolver;
+import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
+
+import org.junit.Test;
+
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+
+public class TaggingTest extends AbstractTaggingResolverTest {
+
+    @Test
+    public void basicTaggingAlexNet() {
+        TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
+        EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.Alexnet", EMAComponentSymbol.KIND)
+                .orElse(null);
+
+        Collection<TagSymbol> tags = tagging.getTags(symbol, DataPathSymbol.KIND);
+        assertEquals(1, tags.size());
+
+        DataPathSymbol tag = (DataPathSymbol) tags.iterator().next();
+        assertEquals(tag.getPath(), "data");
+        assertEquals(tag.getType(), "random");
+    }
+
+    @Test
+    public void instanceTaggingParent() {
+        TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources/");
+        EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>
+                resolve("tagging.Parent", EMAComponentSymbol.KIND).get();
+
+        Collection<TagSymbol> tagsA1 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("a1").iterator().next(), DataPathSymbol.KIND);
+        assertEquals(1, tagsA1.size());
+
+        DataPathSymbol tagA1 = (DataPathSymbol) tagsA1.iterator().next();
+        assertEquals(tagA1.getPath(), "src/test/models/");
+        assertEquals(tagA1.getType(), "LMDB");
+
+
+        Collection<TagSymbol> tagsA2 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("a2").iterator().next(), DataPathSymbol.KIND);
+        assertEquals(1, tagsA2.size());
+
+        DataPathSymbol tagA2 = (DataPathSymbol) tagsA2.iterator().next();
+        assertEquals(tagA2.getPath(), "lisjef");
+        assertEquals(tagA2.getType(), "r34");
+    }
+}

src/test/resources/tagging/AlexNet.tag

+package tagging;
+conforms to dltag.DataPathTagSchema;
+
+tags AlexNet {
+tag Alexnet with DataPath = {path = data, type = random};
+tag Parent.a1 with DataPath = {path = src/test/models/, type = LMDB};
+tag Parent.a2 with DataPath = {path = lisjef, type = r34};
+}

src/test/resources/tagging/Alexnet.emadl

+package tagging;
+
+component Alexnet{
+    ports in Z(0:255)^{3, 224, 224} image,
+         out Q(0:1)^{1000} predictions;
+
+    implementation CNN {
+
+        def split1(i){
+            [i] ->
+            Convolution(kernel=(5,5), channels=128) ->
+            Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()
+        }
+        def split2(i){
+            [i] ->
+            Convolution(kernel=(3,3), channels=192) ->
+            Relu() ->
+            Convolution(kernel=(3,3), channels=128) ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
+            Relu()