Commit 7ca84024 authored by Evgeny Kusmenko's avatar Evgeny Kusmenko
Browse files

Merge branch 'feature/path_tagging' into 'master'

Added Datapath-Tagging for Deep Learning Components

See merge request !15
parents 462cc9c7 ed2b2df9
Pipeline #163781 passed with stages
in 4 minutes and 25 seconds
......@@ -30,7 +30,7 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-deeplearning</artifactId>
<version>0.2.8-SNAPSHOT</version>
<version>0.2.9-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl._cocos;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
import de.se_rwth.commons.logging.Log;
import java.io.File;
public class DataPathCocos {
public static void check(DataPathSymbol dataPathSymbol) {
File dataPath = new File(dataPathSymbol.getPath());
if (!dataPath.exists()) {
Log.warn(String.format("Filepath '%s' does not exist!", dataPath.getAbsolutePath()));
//Log.error(String.format("Filepath '%s' does not exist!", dataPath.getAbsolutePath()));
// TODO should be error, but test exits abruptly if set to error
}
if (!dataPathSymbol.getType().equals("HDF5") && !dataPathSymbol.getType().equals("LMDB")) {
Log.warn("DatapathType is incorrect, must be of Type: HDF5 or LMDB");
//Log.error("DatapathType is incorrect, must be of Type: HDF5 or LMDB");
// TODO should be error, but test exits abruptly if set to error
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.tagging.dltag;
import de.monticore.lang.tagging._symboltable.TagKind;
import de.monticore.lang.tagging._symboltable.TagSymbol;
public class DataPathSymbol extends TagSymbol {
public static final DataPathKind KIND = DataPathKind.INSTANCE;
public DataPathSymbol() {
super(KIND, ".");
}
public DataPathSymbol(String path, String type) {
this(KIND, path, type);
}
public DataPathSymbol(DataPathKind kind, String path, String type) {
super(kind, path, type);
}
public String getPath() {
return getValue(0);
}
public String getType() {
return getValue(1);
}
@Override
public String toString() {
return super.toString();
}
public static class DataPathKind extends TagKind {
public static final DataPathKind INSTANCE = new DataPathKind();
protected DataPathKind() {
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.tagging.dltag;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.embeddedmontiarcdynamic.embeddedmontiarcdynamic._symboltable.instanceStructure.EMADynamicComponentInstantiationSymbol;
import de.monticore.lang.tagging._ast.ASTNameScope;
import de.monticore.lang.tagging._ast.ASTTag;
import de.monticore.lang.tagging._ast.ASTTagBody;
import de.monticore.lang.tagging._ast.ASTTaggingUnit;
import de.monticore.lang.tagging._symboltable.TagSymbolCreator;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.SymbolKind;
import de.monticore.types.types._ast.ASTQualifiedName;
import de.se_rwth.commons.Joiners;
import de.se_rwth.commons.logging.Log;
import org.apache.commons.lang3.StringUtils;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DataPathSymbolCreator implements TagSymbolCreator {
private final String regexExpression = "\\s*\\{\\s*path\\s*=\\s*(.*)\\s*,\\s*type\\s*=\\s*(.*)\\s*\\}\\s*";
private final Pattern pattern = Pattern.compile(regexExpression, Pattern.MULTILINE);
@Override
public void create(ASTTaggingUnit unit, TaggingResolver tagging) {
boolean hasDataPathTag =
unit.getQualifiedNameList().stream()
.map(ASTQualifiedName::toString)
.anyMatch(n -> n.endsWith("DataPathTagSchema"));
if (!hasDataPathTag) {
return;
}
final String packageName = Joiners.DOT.join(unit.getPackageList());
final ASTTagBody tagBody = unit.getTagBody();
final String root =
(tagBody.getTargetModelOpt().isPresent()) ?
Joiners.DOT.join(packageName, tagBody.getTargetModelOpt().get()
.getQualifiedNameString()) :
packageName;
for (ASTTag tag : unit.getTagBody().getTagList()) {
addTag(tag, tagging, root, EMADynamicComponentInstantiationSymbol.KIND);
addTag(tag, tagging, root, EMAComponentSymbol.KIND);
}
}
private void addTag(ASTTag tag, TaggingResolver tagging, String root, SymbolKind kind) {
tag.getTagElementList().stream()
.filter(tagElement -> tagElement.getName().equals("DataPath"))
.forEachOrdered(tagElement -> {
Matcher matcher = matchRegexPattern(tagElement.getTagValue());
tag.getScopeList().stream()
.filter(scope -> scope.getScopeKind().equals("NameScope"))
.map(scope -> (ASTNameScope) scope)
.map(scope ->
tagging.resolve(dotJoin(root, scope.getQualifiedNameString()), kind)
)
.filter(Optional::isPresent)
.map(Optional::get)
.forEachOrdered(scope ->
tagging.addTag(scope, new DataPathSymbol(matcher.group(1), matcher.group(2)))
);
});
}
private Matcher matchRegexPattern(String regex) {
Matcher matcher = pattern.matcher(regex);
if (matcher.matches()) {
return matcher;
}
else {
Log.error(
String.format(
"'%s' does not match the specified regex pattern '%s'",
regex, "{path = {dataPath}, type = {dataType}}"
)
);
return null;
}
}
private String dotJoin(String root, String name) {
if (StringUtils.isEmpty(root)) {
return name;
}
else {
return Joiners.DOT.join(root, name);
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.tagging.dltag;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.resolving.CommonResolvingFilter;
public class DataPathTagSchema {
protected static DataPathTagSchema instance = null;
protected DataPathTagSchema() {
}
protected static DataPathTagSchema getInstance() {
if (instance == null) {
instance = new DataPathTagSchema();
}
return instance;
}
protected void doRegisterTagTypes(TaggingResolver tagging) {
tagging.addTagSymbolCreator(new DataPathSymbolCreator());
tagging.addTagSymbolResolvingFilter(CommonResolvingFilter.create(DataPathSymbol.KIND));
}
public static void registerTagTypes(TaggingResolver tagging) {
getInstance().doRegisterTagTypes(tagging);
}
}
package dltag;
tagschema DeepLearningFilePathSchema {
tagtype DataPath for EMAComponentSymbol is { path = ${path:String}, type = ${type:String} };
}
\ No newline at end of file
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathTagSchema;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.Scope;
import java.util.Arrays;
public class AbstractTaggingResolverTest extends AbstractSymtabTest {
protected static TaggingResolver createSymTabandTaggingResolver(String... modelPath) {
Scope scope = createSymTab(modelPath);
TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));
DataPathTagSchema.registerTagTypes(tagging);
return tagging;
}
}
......@@ -85,6 +85,7 @@ public class AbstractCoCoTest extends AbstractSymtabTest {
// check whether all the expected errors are present when using all cocos
Log.getFindings().clear();
EMADLCocos.checkAll(getComponentInstance(modelPath, model));
EMADLCocos.checkAll(getComponentInstance(modelPath, model));
expectedErrors.checkExpectedPresent(Log.getFindings(), "Got no findings when checking all "
+ "cocos. Did you forget to add the new coco to MontiArcCocos?");
}
......
......@@ -65,4 +65,4 @@ public class AllCoCoTest extends AbstractCoCoTest {
checkValid("models", "MinimizePortsTest");
}
}
\ No newline at end of file
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.cocos;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.embeddedmontiarcdynamic.embeddedmontiarcdynamic._symboltable.instanceStructure.EMADynamicComponentInstantiationSymbol;
import de.monticore.lang.tagging._symboltable.TagSymbol;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
import de.monticore.lang.monticar.cnnarch.helper.ErrorCodes;
import de.monticore.lang.monticar.emadl._cocos.DataPathCocos;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import org.junit.Before;
import org.junit.Test;
import java.util.Collection;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import de.monticore.lang.monticar.emadl.AbstractTaggingResolverTest;
import static de.monticore.lang.monticar.emadl.ParserTest.ENABLE_FAIL_QUICK;
public class TaggingCoCoTest extends AbstractTaggingResolverTest {
@Before
public void setUp() {
// ensure an empty log
Log.getFindings().clear();
Log.enableFailQuick(ENABLE_FAIL_QUICK);
}
@Test
public void testCoCosWithInvalidType() {
TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.Alexnet", EMAComponentSymbol.KIND)
.orElse(null);
assertNotNull(symbol);
Collection<TagSymbol> tags = tagging.getTags(symbol, DataPathSymbol.KIND);
assertEquals(1, tags.size());
DataPathSymbol tag = (DataPathSymbol) tags.iterator().next();
checkValid(tag);
assertFalse(Log.getFindings().isEmpty());
}
@Test
public void testCoCosWithValidType() {
TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.CorrectTypeNet", EMAComponentSymbol.KIND)
.orElse(null);
assertNotNull(symbol);
Collection<TagSymbol> tags = tagging.getTags(symbol, DataPathSymbol.KIND);
assertEquals(1, tags.size());
DataPathSymbol tag = (DataPathSymbol) tags.iterator().next();
checkValid(tag);
assertTrue(Log.getFindings().isEmpty());
}
@Test
public void testCoCosForInstancesWithValidType() {
TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources/");
EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.CorrectTypeInstance", EMAComponentSymbol.KIND)
.orElse(null);
assertNotNull(symbol);
Collection<TagSymbol> tagsNet1 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("net1").iterator().next(), DataPathSymbol.KIND);
assertEquals(1, tagsNet1.size());
checkValid((DataPathSymbol) tagsNet1.iterator().next());
Collection<TagSymbol> tagsNet2 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("net2").iterator().next(), DataPathSymbol.KIND);
assertEquals(1, tagsNet2.size());
checkValid((DataPathSymbol) tagsNet2.iterator().next());
assertTrue(Log.getFindings().isEmpty());
}
protected static void checkValid(DataPathSymbol dataPathSymbol) {
Log.getFindings().clear();
DataPathCocos.check(dataPathSymbol);
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.tagging.dltag;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.emadl.AbstractTaggingResolverTest;
import de.monticore.lang.tagging._symboltable.TagSymbol;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathSymbol;
import org.junit.Test;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
public class TaggingTest extends AbstractTaggingResolverTest {
@Test
public void basicTaggingAlexNet() {
TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>resolve("tagging.Alexnet", EMAComponentSymbol.KIND)
.orElse(null);
Collection<TagSymbol> tags = tagging.getTags(symbol, DataPathSymbol.KIND);
assertEquals(1, tags.size());
DataPathSymbol tag = (DataPathSymbol) tags.iterator().next();
assertEquals(tag.getPath(), "data");
assertEquals(tag.getType(), "random");
}
@Test
public void instanceTaggingParent() {
TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources/");
EMAComponentSymbol symbol = tagging.<EMAComponentSymbol>
resolve("tagging.Parent", EMAComponentSymbol.KIND).get();
Collection<TagSymbol> tagsA1 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("a1").iterator().next(), DataPathSymbol.KIND);
assertEquals(1, tagsA1.size());
DataPathSymbol tagA1 = (DataPathSymbol) tagsA1.iterator().next();
assertEquals(tagA1.getPath(), "src/test/models/");
assertEquals(tagA1.getType(), "LMDB");
Collection<TagSymbol> tagsA2 = tagging.getTags(symbol.getSpannedScope().getLocalSymbols().get("a2").iterator().next(), DataPathSymbol.KIND);
assertEquals(1, tagsA2.size());
DataPathSymbol tagA2 = (DataPathSymbol) tagsA2.iterator().next();
assertEquals(tagA2.getPath(), "lisjef");
assertEquals(tagA2.getType(), "r34");
}
}
package tagging;
conforms to dltag.DataPathTagSchema;
tags AlexNet {
tag Alexnet with DataPath = {path = data, type = random};
tag Parent.a1 with DataPath = {path = src/test/models/, type = LMDB};
tag Parent.a2 with DataPath = {path = lisjef, type = r34};
}
package tagging;
component Alexnet{
ports in Z(0:255)^{3, 224, 224} image,
out Q(0:1)^{1000} predictions;
implementation CNN {
def split1(i){
[i] ->
Convolution(kernel=(5,5), channels=128) ->
Lrn(nsize=5, alpha=0.0001, beta=0.75) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()
}
def split2(i){
[i] ->
Convolution(kernel=(3,3), channels=192) ->
Relu() ->
Convolution(kernel=(3,3), channels=128) ->
Pooling(pool_type="max", kernel=(3,3), stride=(2,2), padding="no_loss") ->
Relu()