Commit 17ae6b33 authored by danielkisov's avatar danielkisov
Browse files

Merge remote-tracking branch 'origin/master' into ba_kisov

parents 2c4c4367 f02bd625
......@@ -39,7 +39,7 @@ masterJobLinux:
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean deploy -U --settings settings.xml
- cat target/site/jacoco/index.html
- mvn package sonar:sonar -s settings.xml
- mvn package -s settings.xml
only:
- master
except:
......
/**
* (c) https://github.com/MontiCore/monticore
*
* The license generally applicable for this project
* can be found under https://github.com/MontiCore/monticore.
*/
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.tagging._symboltable.TagKind;
import de.monticore.lang.tagging._symboltable.TagSymbol;
public class DatasetArtifactSymbol extends TagSymbol {
public static final DatasetArtifactKind KIND = DatasetArtifactKind.INSTANCE;
public DatasetArtifactSymbol() {
super(KIND, ".");
}
public DatasetArtifactSymbol(String artifact, String jar, String type) {
this(KIND, artifact, jar, type);
}
public DatasetArtifactSymbol(DatasetArtifactKind kind, String artifact, String jar, String type) {
super(kind, artifact, jar, type);
}
public String getArtifact() {
return getValue(0);
}
public String getJar() {
return getValue(1);
}
public String getType() {
return getValue(2);
}
@Override
public String toString() {
return super.toString();
}
public static class DatasetArtifactKind extends TagKind {
public static final DatasetArtifactKind INSTANCE = new DatasetArtifactKind();
protected DatasetArtifactKind() {}
}
}
/**
* (c) https://github.com/MontiCore/monticore
*
* The license generally applicable for this project
* can be found under https://github.com/MontiCore/monticore.
*/
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.embeddedmontiarcdynamic.embeddedmontiarcdynamic._symboltable.instanceStructure.EMADynamicComponentInstantiationSymbol;
import de.monticore.lang.tagging._ast.ASTNameScope;
import de.monticore.lang.tagging._ast.ASTTag;
import de.monticore.lang.tagging._ast.ASTTagBody;
import de.monticore.lang.tagging._ast.ASTTaggingUnit;
import de.monticore.lang.tagging._symboltable.TagSymbolCreator;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.SymbolKind;
import de.monticore.types.types._ast.ASTQualifiedName;
import de.se_rwth.commons.Joiners;
import de.se_rwth.commons.logging.Log;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DatasetArtifactSymbolCreator implements TagSymbolCreator {
protected final String regexExpression = "\\s*\\{\\s*artifact\\s*=\\s*(.+):(.+):(.+)\\s*,\\s*type\\s*=\\s*(.*)\\s*\\}\\s*";
protected final Pattern pattern = Pattern.compile(regexExpression, Pattern.MULTILINE);
@Override
public void create(ASTTaggingUnit taggingUnit, TaggingResolver resolver) {
boolean hasLayerArtifactParameterTag =
taggingUnit.getQualifiedNameList().stream()
.map(ASTQualifiedName::toString)
.anyMatch(n -> n.endsWith("DatasetArtifactTagSchema"));
if (!hasLayerArtifactParameterTag) {
return;
}
final String packageName = Joiners.DOT.join(taggingUnit.getPackageList());
final ASTTagBody tagBody = taggingUnit.getTagBody();
final String root =
(tagBody.getTargetModelOpt().isPresent()) ?
Joiners.DOT.join(packageName, tagBody.getTargetModelOpt().get()
.getQualifiedNameString()) :
packageName;
for (ASTTag tag : taggingUnit.getTagBody().getTagList()) {
addTag(tag, resolver, root, EMADynamicComponentInstantiationSymbol.KIND);
addTag(tag, resolver, root, EMAComponentSymbol.KIND);
}
}
protected void addTag(ASTTag tag, TaggingResolver tagging, String root, SymbolKind kind) {
tag.getTagElementList().stream()
.filter(tagElement -> tagElement.getName().equals("DatasetArtifact"))
.forEachOrdered(tagElement -> {
Matcher matcher = matchRegexPattern(tagElement.getTagValue());
tag.getScopeList().stream()
.filter(scope -> scope.getScopeKind().equals("NameScope"))
.map(scope -> (ASTNameScope) scope)
.map(scope ->
tagging.resolve(dotJoin(root, scope.getQualifiedNameString()), kind)
)
.filter(Optional::isPresent)
.map(Optional::get)
.forEachOrdered(scope -> {
String artifact = String.format("%s%s%s%s%s", matcher.group(1).replace('.', File.separatorChar),
File.separator, matcher.group(2), File.separator, matcher.group(3));
String jar = String.format("%s-%s-dataset", matcher.group(2), matcher.group(3));
tagging.addTag(scope, new DatasetArtifactSymbol(artifact, jar, matcher.group(4))); }
);
});
}
protected Matcher matchRegexPattern(String regex) {
Matcher matcher = pattern.matcher(regex);
if (matcher.matches()) {
return matcher;
}
else {
Log.error(
String.format(
"'%s' does not match the specified regex pattern '%s'",
regex, "{artifact = {groupId:artifactId:version}}"
)
);
return null;
}
}
protected String dotJoin(String root, String name) {
if (StringUtils.isEmpty(root)) {
return name;
}
else {
return Joiners.DOT.join(root, name);
}
}
}
/**
* (c) https://github.com/MontiCore/monticore
*
* The license generally applicable for this project
* can be found under https://github.com/MontiCore/monticore.
*/
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.resolving.CommonResolvingFilter;
public class DatasetArtifactTagSchema {
protected static DatasetArtifactTagSchema instance = null;
protected DatasetArtifactTagSchema() {}
protected static DatasetArtifactTagSchema getInstance() {
if (instance == null) {
instance = new DatasetArtifactTagSchema();
}
return instance;
}
protected void doRegisterTagTypes(TaggingResolver resolver) {
resolver.addTagSymbolCreator(new DatasetArtifactSymbolCreator());
resolver.addTagSymbolResolvingFilter(CommonResolvingFilter.create(DatasetArtifactSymbol.KIND));
}
public static void registerTagTypes(TaggingResolver resolver) {
getInstance().doRegisterTagTypes(resolver);
}
}
/**
* (c) https://github.com/MontiCore/monticore
*
* The license generally applicable for this project
* can be found under https://github.com/MontiCore/monticore.
*/
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.tagging._symboltable.TagKind;
import de.monticore.lang.tagging._symboltable.TagSymbol;
public class LayerArtifactParameterSymbol extends TagSymbol {
public static final LayerArtifactParameterKind KIND = LayerArtifactParameterKind.INSTANCE;
public LayerArtifactParameterSymbol() {
super(KIND, ".");
}
public LayerArtifactParameterSymbol(String artifact, String jar, String id) {
this(KIND, artifact, jar, id);
}
public LayerArtifactParameterSymbol(LayerArtifactParameterKind kind, String artifact, String jar, String id) {
super(kind, artifact, jar, id);
}
public String getArtifact() {
return getValue(0);
}
public String getJar() {
return getValue(1);
}
public String getId() {
return getValue(2);
}
@Override
public String toString() {
return super.toString();
}
public static class LayerArtifactParameterKind extends TagKind {
public static final LayerArtifactParameterKind INSTANCE = new LayerArtifactParameterKind();
protected LayerArtifactParameterKind() {
}
}
}
/**
* (c) https://github.com/MontiCore/monticore
*
* The license generally applicable for this project
* can be found under https://github.com/MontiCore/monticore.
*/
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.embeddedmontiarcdynamic.embeddedmontiarcdynamic._symboltable.instanceStructure.EMADynamicComponentInstantiationSymbol;
import de.monticore.lang.tagging._ast.ASTNameScope;
import de.monticore.lang.tagging._ast.ASTTag;
import de.monticore.lang.tagging._ast.ASTTagBody;
import de.monticore.lang.tagging._ast.ASTTaggingUnit;
import de.monticore.lang.tagging._symboltable.TagSymbolCreator;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.SymbolKind;
import de.monticore.types.types._ast.ASTQualifiedName;
import de.se_rwth.commons.Joiners;
import de.se_rwth.commons.logging.Log;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class LayerArtifactParameterSymbolCreator implements TagSymbolCreator {
protected final String regexExpression = "\\s*\\{\\s*artifact\\s*=\\s*(.+):(.+):(.+)\\s*,\\s*id\\s*=\\s*(.*)\\s*\\}\\s*";
protected final Pattern pattern = Pattern.compile(regexExpression, Pattern.MULTILINE);
@Override
public void create(ASTTaggingUnit unit, TaggingResolver tagging) {
boolean hasLayerArtifactParameterTag =
unit.getQualifiedNameList().stream()
.map(ASTQualifiedName::toString)
.anyMatch(n -> n.endsWith("LayerArtifactParameterTagSchema"));
if (!hasLayerArtifactParameterTag) {
return;
}
final String packageName = Joiners.DOT.join(unit.getPackageList());
final ASTTagBody tagBody = unit.getTagBody();
final String root =
(tagBody.getTargetModelOpt().isPresent()) ?
Joiners.DOT.join(packageName, tagBody.getTargetModelOpt().get()
.getQualifiedNameString()) :
packageName;
for (ASTTag tag : unit.getTagBody().getTagList()) {
addTag(tag, tagging, root, EMADynamicComponentInstantiationSymbol.KIND);
addTag(tag, tagging, root, EMAComponentSymbol.KIND);
}
}
protected void addTag(ASTTag tag, TaggingResolver tagging, String root, SymbolKind kind) {
tag.getTagElementList().stream()
.filter(tagElement -> tagElement.getName().equals("LayerArtifactParameter"))
.forEachOrdered(tagElement -> {
Matcher matcher = matchRegexPattern(tagElement.getTagValue());
tag.getScopeList().stream()
.filter(scope -> scope.getScopeKind().equals("NameScope"))
.map(scope -> (ASTNameScope) scope)
.map(scope ->
tagging.resolve(dotJoin(root, scope.getQualifiedNameString()), kind)
)
.filter(Optional::isPresent)
.map(Optional::get)
.forEachOrdered(scope -> {
String artifact = String.format("%s%s%s%s%s", matcher.group(1).replace('.', File.separatorChar),
File.separator, matcher.group(2), File.separator, matcher.group(3));
String jar = String.format("%s-%s-pretrained", matcher.group(2), matcher.group(3));
tagging.addTag(scope, new LayerArtifactParameterSymbol(artifact, jar, matcher.group(4))); }
);
});
}
protected Matcher matchRegexPattern(String regex) {
Matcher matcher = pattern.matcher(regex);
if (matcher.matches()) {
return matcher;
}
else {
Log.error(
String.format(
"'%s' does not match the specified regex pattern '%s'",
regex, "{artifact = {groupId:artifactId:version}, id = {tagId}}"
)
);
return null;
}
}
protected String dotJoin(String root, String name) {
if (StringUtils.isEmpty(root)) {
return name;
}
else {
return Joiners.DOT.join(root, name);
}
}
}
/**
* (c) https://github.com/MontiCore/monticore
*
* The license generally applicable for this project
* can be found under https://github.com/MontiCore/monticore.
*/
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.resolving.CommonResolvingFilter;
public class LayerArtifactParameterTagSchema {
protected static LayerArtifactParameterTagSchema instance = null;
protected LayerArtifactParameterTagSchema() {
}
protected static LayerArtifactParameterTagSchema getInstance() {
if (instance == null) {
instance = new LayerArtifactParameterTagSchema();
}
return instance;
}
protected void doRegisterTagTypes(TaggingResolver tagging) {
tagging.addTagSymbolCreator(new LayerArtifactParameterSymbolCreator());
tagging.addTagSymbolResolvingFilter(CommonResolvingFilter.create(LayerArtifactParameterSymbol.KIND));
}
public static void registerTagTypes(TaggingResolver tagging) {
getInstance().doRegisterTagTypes(tagging);
}
}
......@@ -31,7 +31,7 @@ public class DataPathTagSchema {
}
public static void registerTagTypes(TaggingResolver tagging) {
getInstance().doRegisterTagTypes(tagging);
getInstance().doRegisterTagTypes(tagging);
}
}
/* (c) https://github.com/MontiCore/monticore */
package artifacttag;
tagschema DatasetArtifactTagSchema {
tagtype DatasetArtifact for EMAComponentSymbol is
{ artifact = ${artifact:String}, type = ${type:String} };
tagtype DatasetArtifact for EMAComponentInstanceSymbol is
{ artifact = ${artifact:String}, type = ${type:String} };
}
\ No newline at end of file
/* (c) https://github.com/MontiCore/monticore */
package artifacttag;
tagschema LayerArtifactParameterTagSchema {
tagtype LayerArtifactParameter for EMAComponentSymbol is
{ artifact = ${artifact:String}, id = ${id:String} };
tagtype LayerArtifactParameter for EMAComponentInstanceSymbol is
{ artifact = ${artifact:String}, id = ${id:String} };
}
\ No newline at end of file
......@@ -7,6 +7,8 @@
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.tagging.artifacttag.DatasetArtifactTagSchema;
import de.monticore.lang.monticar.emadl.tagging.artifacttag.LayerArtifactParameterTagSchema;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathTagSchema;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.Scope;
......@@ -19,6 +21,8 @@ public class AbstractTaggingResolverTest extends AbstractSymtabTest {
Scope scope = createSymTab(modelPath);
TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));
DataPathTagSchema.registerTagTypes(tagging);
DatasetArtifactTagSchema.registerTagTypes(tagging);
LayerArtifactParameterTagSchema.registerTagTypes(tagging);
return tagging;
}
......
package de.monticore.lang.monticar.emadl.tagging.artifacttag;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAComponentSymbol;
import de.monticore.lang.monticar.emadl.AbstractTaggingResolverTest;
import de.monticore.lang.tagging._symboltable.TagSymbol;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import org.junit.Test;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
public class ArtifactTaggingTest extends AbstractTaggingResolverTest {
private TaggingResolver tagging = createSymTabandTaggingResolver("src/test/resources");
private EMAComponentSymbol parentSymbol =
tagging.<EMAComponentSymbol>resolve("tagging.artifacttagging.Parent", EMAComponentSymbol.KIND).orElse(null);
private EMAComponentSymbol alexnetSymbol =
tagging.<EMAComponentSymbol>resolve("tagging.artifacttagging.Alexnet", EMAComponentSymbol.KIND).orElse(null);
@Test
public void testValidDatasetArtifactTagForComponent() {
Collection<TagSymbol> tags = tagging.getTags(alexnetSymbol, DatasetArtifactSymbol.KIND);
assertEquals(1, tags.size());
DatasetArtifactSymbol datasetArtifactSymbol = (DatasetArtifactSymbol) tags.iterator().next();
assertEquals("com/emadl/dataset/mnist/2", datasetArtifactSymbol.getArtifact());
assertEquals("mnist-2-dataset", datasetArtifactSymbol.getJar());
assertEquals("HDF5", datasetArtifactSymbol.getType());
}
@Test
public void testValidDatasetArtifactTagForComponentForInstances() {
Collection<TagSymbol> tagsA1 = tagging.getTags(parentSymbol.getSpannedScope().getLocalSymbols().get("a1").iterator().next(), DatasetArtifactSymbol.KIND);
Collection<TagSymbol> tagsA2 = tagging.getTags(parentSymbol.getSpannedScope().getLocalSymbols().get("a2").iterator().next(), DatasetArtifactSymbol.KIND);
assertEquals(1, tagsA1.size());
assertEquals(1, tagsA2.size());
DatasetArtifactSymbol tagA1 = (DatasetArtifactSymbol) tagsA1.iterator().next();
assertEquals("com/emadl/dataset/sst2/40", tagA1.getArtifact());
assertEquals("sst2-40-dataset", tagA1.getJar());
assertEquals("LMDB", tagA1.getType());
DatasetArtifactSymbol tagA2 = (DatasetArtifactSymbol) tagsA2.iterator().next();
assertEquals("com/monticore/lang/monticar/imdb/2", tagA2.getArtifact());
assertEquals("imdb-2-dataset", tagA2.getJar());
assertEquals("HDF5", tagA2.getType());
}
@Test
public void testValidLayerArtifactParameterTagForComponent() {
Collection<TagSymbol> tags = tagging.getTags(alexnetSymbol, LayerArtifactParameterSymbol.KIND);
assertEquals(1, tags.size());
LayerArtifactParameterSymbol layerArtifactParameterSymbol = (LayerArtifactParameterSymbol) tags.iterator().next();
assertEquals("com/emadl/pretrained-model/bert-small/2", layerArtifactParameterSymbol.getArtifact());
assertEquals("bert-small-2-pretrained", layerArtifactParameterSymbol.getJar());
assertEquals("bert-small", layerArtifactParameterSymbol.getId());
}
@Test
public void testValidLayerArtifactParameterTagForComponentForInstances() {
Collection<TagSymbol> tagsA1 = tagging.getTags(parentSymbol.getSpannedScope().getLocalSymbols().get("a1").iterator().next(), LayerArtifactParameterSymbol.KIND);
assertEquals(1, tagsA1.size());
LayerArtifactParameterSymbol layerArtifactParameterSymbolA1 = (LayerArtifactParameterSymbol) tagsA1.iterator().next();
assertEquals("com/emadl/pretrained-model/bert-large/1", layerArtifactParameterSymbolA1.getArtifact());
assertEquals("bert-large-1-pretrained", layerArtifactParameterSymbolA1.getJar());
assertEquals("bert", layerArtifactParameterSymbolA1.getId());
}
}
/* (c) https://github.com/MontiCore/monticore */
package tagging.artifacttagging;
conforms to artifacttag.DatasetArtifactTagSchema,
artifacttag.LayerArtifactParameterTagSchema;
tags AlexNet {
// DatasetArtifact Tags
tag Alexnet with DatasetArtifact = {artifact = com.emadl.dataset:mnist:2, type = HDF5};
tag Parent.a1 with DatasetArtifact = {artifact = com.emadl.dataset:sst2:40, type = LMDB};
tag Parent.a2 with DatasetArtifact = {artifact = com.monticore.lang.monticar:imdb:2, type = HDF5};
// LayerArtifactParameter Tags
tag Alexnet with LayerArtifactParameter = {artifact = com.emadl.pretrained-model:bert-small:2, id = bert-small};
tag Parent.a1 with LayerArtifactParameter = {artifact = com.emadl.pretrained-model:bert-large:1, id = bert};
}
/* (c) https://github.com/MontiCore/monticore */
package tagging.artifacttagging;
component Alexnet{
ports in Z(0:255)^{3, 224, 224} image,