Commit 976ee231 authored by Kirhan, Cihad's avatar Kirhan, Cihad Committed by C. Kirhan
Browse files

Integration of ConfLang and SchemaLang

parent c2eaa336
Pipeline #591125 passed with stage
in 1 minute and 1 second
......@@ -3,6 +3,7 @@
stages:
#- windows
- linux
#- deploy
githubjob:
image: maven:3-jdk-8
......@@ -56,3 +57,9 @@ BranchJobLinux:
- cat target/site/jacoco/index.html
except:
- master
#deploy:
# stage: linux
# image: maven:3-jdk-8
# script:
# - "mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean deploy --settings settings.xml"
\ No newline at end of file
......@@ -2,25 +2,22 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<modelVersion>4.0.0</modelVersion>
<!-- == PROJECT COORDINATES ============================================= -->
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-generator</artifactId>
<version>0.4.10-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<CNNArch.version>0.4.8-SNAPSHOT</CNNArch.version>
<conflang.version>1.0.0-SNAPSHOT</conflang.version>
<schemalang.version>1.0.0-SNAPSHOT</schemalang.version>
<embedded-montiarc-math-generator>0.4.9</embedded-montiarc-math-generator>
<EMADL2PythonWrapper.version>0.0.3-SNAPSHOT</EMADL2PythonWrapper.version>
<!-- == PROJECT COORDINATES ============================================= -->
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-generator</artifactId>
<version>0.4.10-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<CNNArch.version>0.4.8-SNAPSHOT</CNNArch.version>
<CNNTrain.version>0.4.5-SNAPSHOT</CNNTrain.version>
<embedded-montiarc-math-generator>0.4.9</embedded-montiarc-math-generator>
<!-- .. Libraries .................................................. -->
<guava.version>25.1-jre</guava.version>
<junit.version>4.13.1</junit.version>
......@@ -34,16 +31,16 @@
<shade.plugin>2.4.3</shade.plugin>
<jacoco.plugin>0.8.1</jacoco.plugin>
<!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier>
<cli.classifier>cli</cli.classifier>
<!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier>
<cli.classifier>cli</cli.classifier>
<!-- .. Misc .......................................................... -->
<java.version>1.8</java.version>
<!-- .. Misc .......................................................... -->
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
......@@ -68,9 +65,15 @@
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnn-train</artifactId>
<version>${CNNTrain.version}</version>
<groupId>de.monticore.lang</groupId>
<artifactId>conflang</artifactId>
<version>${conflang.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang</groupId>
<artifactId>schemalang</artifactId>
<version>${schemalang.version}</version>
</dependency>
<dependency>
......@@ -79,6 +82,11 @@
<version>${embedded-montiarc-math-generator}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-pythonwrapper-generator</artifactId>
<version>${EMADL2PythonWrapper.version}</version>
</dependency>
<!-- .. Test Libraries ............................................... -->
<dependency>
......@@ -107,97 +115,97 @@
</dependency>
</dependencies>
<!-- == PROJECT BUILD SETTINGS =========================================== -->
<build>
<plugins>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.1</version>
</plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>${compiler.plugin}</version>
<configuration>
<useIncrementalCompilation>true</useIncrementalCompilation>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>jar-with-dependencies</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<archive>
<manifest>
<!-- == PROJECT BUILD SETTINGS =========================================== -->
<build>
<plugins>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.1</version>
</plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>${compiler.plugin}</version>
<configuration>
<useIncrementalCompilation>true</useIncrementalCompilation>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>jar-with-dependencies</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<archive>
<manifest>
<mainClass>de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNetCli</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
</executions>
</plugin>
<!-- Source Jar Configuration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${source.plugin}</version>
<executions>
<execution>
<id>create source jar</id>
<phase>package</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
<configuration>
<excludeResources>false</excludeResources>
<includes>
<include>**/*.java</include>
<include>**/*.ftl</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
</executions>
</plugin>
<!-- Source Jar Configuration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${source.plugin}</version>
<executions>
<execution>
<id>create source jar</id>
<phase>package</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
<configuration>
<excludeResources>false</excludeResources>
<includes>
<include>**/*.java</include>
<include>**/*.ftl</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
......@@ -222,9 +230,9 @@
<!-- aggregated reports for multi-module projects -->
<aggregate>true</aggregate>
</configuration>
</plugin>
</plugins>
</build>
</plugin>
</plugins>
</build>
<distributionManagement>
<repository>
......@@ -236,5 +244,4 @@
<url>https://nexus.se.rwth-aachen.de/content/repositories/embeddedmontiarc-snapshots/</url>
</snapshotRepository>
</distributionManagement>
</project>
</project>
\ No newline at end of file
<!-- (c) https://github.com/MontiCore/monticore -->
<assembly
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>schemas</id>
<includeBaseDirectory>false</includeBaseDirectory>
<includeSiteDirectory>false</includeSiteDirectory>
<fileSets>
<fileSet>
<outputDirectory/>
<directory>src/main/schemas</directory>
</fileSet>
</fileSets>
</assembly>
\ No newline at end of file
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.cnnarch.generator;
import conflang._cocos.ConfLangCoCoChecker;
import conflang._cocos.ConfLangCocoFactory;
import conflang._symboltable.ConfLangLanguage;
import conflang._symboltable.ConfigurationSymbol;
import de.monticore.ModelingLanguageFamily;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.monticar.cnnarch.generator.ConfigurationData;
import de.monticore.lang.monticar.cnnarch.generator.TemplateConfiguration;
import de.monticore.lang.monticar.cnntrain._ast.ASTCNNTrainNode;
import de.monticore.lang.monticar.cnntrain._ast.ASTOptimizerEntry;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainCompilationUnitSymbol;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainLanguage;
import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.EmbeddedMontiArcLanguage;
import de.monticore.lang.monticar.cnnarch.generator.training.TrainingComponentsContainer;
import de.monticore.lang.monticar.cnnarch.generator.training.TrainingConfiguration;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cpp.GeneratorCPP;
import de.monticore.symboltable.GlobalScope;
import de.se_rwth.commons.logging.Log;
import schemalang._cocos.SchemaLangCoCoChecker;
import schemalang._cocos.SchemaLangCocoFactory;
import schemalang._symboltable.SchemaDefinitionSymbol;
import schemalang._symboltable.SchemaLangLanguage;
import schemalang.validation.ReferenceModelViolation;
import schemalang.validation.SchemaLangValidator;
import schemalang.validation.SchemaViolation;
import schemalang.validation.Violation;
import schemalang.validation.exception.SchemaLangException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Path;
import java.util.*;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
public abstract class CNNTrainGenerator {
import static de.monticore.lang.monticar.cnnarch.generator.validation.Constants.ROOT_SCHEMA;
import static de.monticore.lang.monticar.cnnarch.generator.validation.Constants.ROOT_SCHEMA_MODEL_PATH;
protected TrainParamSupportChecker trainParamSupportChecker;
public abstract class CNNTrainGenerator {
private TrainParamSupportChecker trainParamSupportChecker;
private String generationTargetPath;
private String instanceName;
protected CNNTrainGenerator() {
public CNNTrainGenerator(TrainParamSupportChecker trainParamSupportChecker) {
setGenerationTargetPath("./target/generated-sources-cnnarch/");
this.trainParamSupportChecker = trainParamSupportChecker;
}
private void supportCheck(ConfigurationSymbol configuration){
checkEntryParams(configuration);
checkOptimizerParams(configuration);
public TrainingConfiguration createTrainingConfiguration(Path modelsDirPath, String rootModelName) {
ConfigurationSymbol configuration = resolveTrainingConfiguration(modelsDirPath, rootModelName);
setInstanceName(rootModelName);
URL schemasResource = getClass().getClassLoader().getResource(ROOT_SCHEMA_MODEL_PATH);
List<SchemaDefinitionSymbol> schemaDefinitionSymbols;
try {
assert schemasResource != null;
Path path = Paths.get(schemasResource.toURI());
ModelPath modelPath = new ModelPath(path);
SchemaDefinitionSymbol schema = resolveSchemaDefinition(ROOT_SCHEMA, modelPath);
SchemaLangCoCoChecker checkerWithAllCoCos = SchemaLangCocoFactory.getCheckerWithAllCoCos();
checkerWithAllCoCos.checkAll(schema.getSchemaDefinitionNode().get());
schemaDefinitionSymbols = SchemaLangValidator.resolveSchemaHierarchy(schema, configuration, modelPath);
List<SchemaViolation> schemaViolations = SchemaLangValidator.validateConfiguration(schemaDefinitionSymbols, configuration);
if (!schemaViolations.isEmpty()) {
logViolations(schemaViolations);
throw new GenerationAbortedException("Generation aborted due to errors in the training configuration.");
}
} catch (SchemaLangException e) {
throw new GenerationAbortedException("Generation aborted due to errors in the training configuration.");
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Generation aborted, since there the path '%s' is not valid.",
schemasResource.toString()));
}
return TrainingConfiguration.create(configuration, schemaDefinitionSymbols);
}
private ConfigurationSymbol resolveTrainingConfiguration(Path modelsDirPath, String rootModelName) {
final ModelPath mp = new ModelPath(modelsDirPath);
GlobalScope scope = new GlobalScope(mp, new ConfLangLanguage());
Optional<ConfigurationSymbol> configurationSymbolOpt = scope.resolve(rootModelName, ConfigurationSymbol.KIND);
if (!configurationSymbolOpt.isPresent()) {
String message = String.format("Could not resolve training configuration for model '%s'.", rootModelName);
Log.error(message);
throw new RuntimeException(message);
}
ConfigurationSymbol configurationSymbol = configurationSymbolOpt.get();
setInstanceName(configurationSymbol.getFullName());
ConfLangCoCoChecker checkerWithAllCoCos = ConfLangCocoFactory.createCheckerWithAllCoCos();
checkerWithAllCoCos.checkAll(configurationSymbol.getConfigurationNode().get());
trainParamSupportChecker.check(configurationSymbolOpt.get());
return configurationSymbolOpt.get();
}
private void checkEntryParams(ConfigurationSymbol configuration){
Iterator it = configuration.getEntryMap().keySet().iterator();
while (it.hasNext()) {
String key = it.next().toString();
ASTCNNTrainNode astTrainEntryNode = (ASTCNNTrainNode) configuration.getEntryMap().get(key).getAstNode().get();
astTrainEntryNode.accept(trainParamSupportChecker);
private SchemaDefinitionSymbol resolveSchemaDefinition(String rootModelName, ModelPath modelPath) {
ModelingLanguageFamily family = new ModelingLanguageFamily();
family.addModelingLanguage(new SchemaLangLanguage());
family.addModelingLanguage(new EmbeddedMontiArcLanguage());
GlobalScope globalScope = new GlobalScope(modelPath, family);
Optional<SchemaDefinitionSymbol> compilationUnit = globalScope.resolve(rootModelName,
SchemaDefinitionSymbol.KIND);
if (!compilationUnit.isPresent()) {
String message = String.format("Could not resolve schema definition for model '%s'.", rootModelName);
Log.error(message);
throw new RuntimeException(message);
}
it = configuration.getEntryMap().keySet().iterator();
while (it.hasNext()) {
String key = it.next().toString();
if (trainParamSupportChecker.getUnsupportedElemList().contains(key)) {
it.remove();
SchemaDefinitionSymbol schemaDefinitionSymbol = compilationUnit.get();
SchemaLangCoCoChecker checkerWithAllCoCos = SchemaLangCocoFactory.getCheckerWithAllCoCos();
checkerWithAllCoCos.checkAll(schemaDefinitionSymbol.getSchemaDefinitionNode().get());
return schemaDefinitionSymbol;
}
protected void validateConfiguration(TrainingConfiguration trainingConfiguration, TrainingComponentsContainer trainingComponentsContainer) {
URL schemasResource = getClass().getClassLoader().getResource("schemas/");
try {
assert schemasResource != null;
Path path = Paths.get(schemasResource.toURI());
Collection<Violation> violations = SchemaLangValidator.validate(trainingConfiguration.getConfigurationSymbol(), ROOT_SCHEMA,
trainingComponentsContainer.getArchitectureComponents(), new ModelPath(path));
if (!violations.isEmpty()) {
logViolations(violations);
throw new GenerationAbortedException("Generation aborted due to errors in the training configuration.");
}
} catch (SchemaLangException e) {
throw new GenerationAbortedException("Generation aborted due to errors in the training configuration.");
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Generation aborted, since there the path '%s' is not valid.", schemasResource.toString()));
}
}
private void checkOptimizerParams(ConfigurationSymbol configuration){
if (configuration.getOptimizer() != null) {
ASTOptimizerEntry astOptimizer = (ASTOptimizerEntry) configuration.getOptimizer().getAstNode().get();
astOptimizer.accept(trainParamSupportChecker);
if (trainParamSupportChecker.getUnsupportedElemList().contains(trainParamSupportChecker.unsupportedOptFlag)) {
configuration.setOptimizer(null);
}else {
Iterator it = configuration.getOptimizer().getOptimizerParamMap().keySet().iterator();
while (it.hasNext()) {
String key = it.next().toString();
if (trainParamSupportChecker.getUnsupportedElemList().contains(key)) {
it.remove();
}
}
private void logViolations(Collection<? extends Violation> violations) {
for (Violation violation : violations) {
if (violation instanceof ReferenceModelViolation) {
ReferenceModelViolation referenceModelViolation = (ReferenceModelViolation) violation;
Log.error(referenceModelViolation.toString());
Collection<Violation> rmViolations = referenceModelViolation.getViolations();
logViolations(rmViolations);
continue;
}
Log.error(violation.toString());
}
}
private static void quitGeneration(){
Log.error("Code generation is aborted");
System.exit(1);
protected void generateFilesFromConfigurationSymbol(TrainingConfiguration trainingConfiguration, TrainingComponentsContainer trainingComponentsContainer) {
List<FileContent> fileContents = generateStrings(trainingConfiguration, trainingComponentsContainer);
GeneratorCPP genCPP = new GeneratorCPP();
genCPP.setGenerationTargetPath(getGenerationTargetPath());
try {
for (FileContent fileContent : fileContents) {
genCPP.generateFile(fileContent);
}
} catch (IOException var8) {
Log.error("CNNTrainer file could not be generated" + var8.getMessage());
}
}
public String getInstanceName() {
......@@ -95,23 +184,13 @@ public abstract class CNNTrainGenerator {
this.generationTargetPath = generationTargetPath;
}
public ConfigurationSymbol getConfigurationSymbol(Path modelsDirPath, String rootModelName) {
final ModelPath mp = new ModelPath(modelsDirPath);
GlobalScope scope = new GlobalScope(mp, new CNNTrainLanguage());
Optional<CNNTrainCompilationUnitSymbol> compilationUnit = scope.resolve(rootModelName, CNNTrainCompilationUnitSymbol.KIND);
if (!compilationUnit.isPresent()) {
Log.error("could not resolve training configuration " + rootModelName);
quitGeneration();
}
setInstanceName(compilationUnit.get().getFullName());
CNNTrainCocos.checkAll(compilationUnit.get());
supportCheck(compilationUnit.get().getConfiguration());
return compilationUnit.get().getConfiguration();
private static void quitGeneration() {
Log.error("Code generation is aborted");
System.exit(1);
}
public abstract void generate(Path modelsDirPath, String rootModelNames);
//check cocos with CNNTrainCocos.checkAll(configuration) before calling this method.
public abstract List<FileContent> generateStrings(ConfigurationSymbol configuration);
}
public abstract List<FileContent> generateStrings(TrainingConfiguration trainingConfiguration, TrainingComponentsContainer trainingComponentsContainer);
}
\ No newline at end of file
package de.monticore.lang.monticar.cnnarch.generator;
import de.se_rwth.commons.logging.Finding;
import java.util.List;
public class GenerationAbortedException extends RuntimeException {
public GenerationAbortedException() {
}
public GenerationAbortedException(String message) {
super(message);
}
public GenerationAbortedException(String message, Throwable cause) {
super(message, cause);
}