Aufgrund einer Störung des s3 Storage, könnten in nächster Zeit folgende GitLab Funktionen nicht zur Verfügung stehen: LFS, Container Registry, Job Artifacs, Uploads (Wiki, Bilder, Projekt-Exporte). Wir bitten um Verständnis. Es wird mit Hochdruck an der Behebung des Problems gearbeitet. Weitere Informationen zur Störung des Object Storage finden Sie hier: https://maintenance.itc.rwth-aachen.de/ticket/status/messages/59-object-storage-pilot

Commit 79d825a6 authored by Kirhan, Cihad's avatar Kirhan, Cihad
Browse files

ConfLang integration

parent f9ccf64e
Pipeline #444650 failed with stage
in 31 seconds
......@@ -2,22 +2,23 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<modelVersion>4.0.0</modelVersion>
<!-- == PROJECT COORDINATES ============================================= -->
<!-- == PROJECT COORDINATES ============================================= -->
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId>
<version>0.4.5</version>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId>
<version>0.4.5</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<properties>
<!-- .. SE-Libraries .................................................. -->
<CNNArch2X.version>0.4.6-SNAPSHOT</CNNArch2X.version>
<conflang.version>0.9.0-SNAPSHOT</conflang.version>
<!-- .. SE-Libraries .................................................. -->
<CNNArch2X.version>0.4.5</CNNArch2X.version>
<!-- .. Libraries .................................................. -->
<guava.version>18.0</guava.version>
<junit.version>4.12</junit.version>
......@@ -31,16 +32,16 @@
<shade.plugin>2.4.3</shade.plugin>
<jacoco.plugin>0.8.1</jacoco.plugin>
<!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier>
<cli.classifier>cli</cli.classifier>
<!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier>
<cli.classifier>cli</cli.classifier>
<!-- .. Misc .......................................................... -->
<java.version>1.8</java.version>
<!-- .. Misc .......................................................... -->
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
......@@ -57,13 +58,17 @@
</dependency>
<!-- MontiCore Dependencies -->
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-generator</artifactId>
<artifactId>cnnarch-generator-conflang</artifactId>
<version>${CNNArch2X.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang</groupId>
<artifactId>conflang</artifactId>
<version>${conflang.version}</version>
</dependency>
<!-- .. Test Libraries ............................................... -->
<dependency>
......@@ -92,97 +97,98 @@
</dependency>
</dependencies>
<!-- == PROJECT BUILD SETTINGS =========================================== -->
<build>
<plugins>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.1</version>
</plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>${compiler.plugin}</version>
<configuration>
<useIncrementalCompilation>true</useIncrementalCompilation>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>jar-with-dependencies</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<archive>
<manifest>
<mainClass>de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNetCli</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
</executions>
</plugin>
<!-- Source Jar Configuration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${source.plugin}</version>
<executions>
<execution>
<id>create source jar</id>
<phase>package</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
<configuration>
<excludeResources>false</excludeResources>
<includes>
<include>**/*.java</include>
<include>**/*.ftl</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<!-- == PROJECT BUILD SETTINGS =========================================== -->
<build>
<plugins>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.1</version>
</plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>${compiler.plugin}</version>
<configuration>
<useIncrementalCompilation>true</useIncrementalCompilation>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>jar-with-dependencies</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<archive>
<manifest>
<mainClass>de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNetCli
</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
</executions>
</plugin>
<!-- Source Jar Configuration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${source.plugin}</version>
<executions>
<execution>
<id>create source jar</id>
<phase>package</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
<configuration>
<excludeResources>false</excludeResources>
<includes>
<include>**/*.java</include>
<include>**/*.ftl</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
......@@ -207,9 +213,9 @@
<!-- aggregated reports for multi-module projects -->
<aggregate>true</aggregate>
</configuration>
</plugin>
</plugins>
</build>
</plugin>
</plugins>
</build>
<distributionManagement>
<repository>
......
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.cnnarch.mxnetgenerator;
import com.google.common.collect.Lists;
import conflang._symboltable.ConfLangConfigurationSymbol;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnnarch.generator.ConfigurationData;
......@@ -49,4 +51,18 @@ public class CNNTrain2MxNet extends CNNTrainGenerator {
fileContents.add(temp);
return fileContents;
}
}
@Override
public List<FileContent> generateFileContents(ConfLangConfigurationSymbol configuration, Map<String, Object> architectureAdapterMap) {
TemplateConfiguration templateConfiguration = new MxNetTemplateConfiguration();
MxNetConfigurationDataConfLang configData = new MxNetConfigurationDataConfLang(configuration, getInstanceName());
List<MxNetConfigurationDataConfLang> configDataList = Lists.newArrayList(configData);
Map<String, Object> ftlContext = Collections.singletonMap("configurations", configDataList);
String templateContent = templateConfiguration.processTemplate(ftlContext, "CNNTrainerConfLang.ftl");
List<FileContent> fileContents = new ArrayList<>();
FileContent temp = new FileContent(templateContent, "CNNTrainerConfLang_" + getInstanceName() + ".py");
fileContents.add(temp);
return fileContents;
}
}
\ No newline at end of file
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.cnnarch.mxnetgenerator;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import conflang._ast.ASTConfigurationEntry;
import conflang._ast.ASTNestedConfigurationEntry;
import conflang._ast.ASTSimpleConfigurationEntry;
import conflang._symboltable.ConfLangConfigurationSymbol;
import conflang._symboltable.NestedConfigurationEntrySymbol;
import conflang._symboltable.SimpleConfigurationEntrySymbol;
import conflangliterals._ast.ASTTypelessLiteral;
import conflangliterals._ast.ASTVectorLiteral;
import de.monticore.lang.monticar.cnntrain._symboltable.LearningMethod;
import de.monticore.literals.literals._ast.*;
import de.monticore.symboltable.Symbol;
import de.monticore.symboltable.SymbolKind;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static de.monticore.lang.monticar.cnnarch.mxnetgenerator.TrainingParameterConstants.*;
public class MxNetConfigurationDataConfLang {
private ConfLangConfigurationSymbol configuration;
private String instanceName;
public MxNetConfigurationDataConfLang(ConfLangConfigurationSymbol configuration, String instanceName) {
this.configuration = configuration;
this.instanceName = instanceName;
}
public Boolean isSupervisedLearning() {
if (configurationContainsKey(LEARNING_METHOD)) {
return getSimpleConfigurationValue(LEARNING_METHOD)
.equals(LearningMethod.SUPERVISED);
}
// TODO: USE DEFAULT FROM SCHEMA
return true;
}
public Boolean isReinforcementLearning() {
if (!configurationContainsKey(LEARNING_METHOD)) {
return false;
}
ASTTypelessLiteral learningMethod = (ASTTypelessLiteral) getSimpleConfigurationValue(LEARNING_METHOD);
return LearningMethod.REINFORCEMENT.toString().equals(learningMethod.getValue());
}
public Integer getBatchSize() {
if (!configurationContainsKey(BATCH_SIZE)) {
return null;
}
ASTSignedIntLiteral batchSizeLiteral = (ASTSignedIntLiteral) getSimpleConfigurationValue(BATCH_SIZE);
return batchSizeLiteral.getValue();
}
public Integer getNumEpoch() {
if (!configurationContainsKey(NUM_EPOCH)) {
return null;
}
ASTSignedIntLiteral batchSizeLiteral = (ASTSignedIntLiteral) getSimpleConfigurationValue(NUM_EPOCH);
return batchSizeLiteral.getValue();
}
public Boolean getLoadCheckpoint() {
if (!configurationContainsKey(LOAD_CHECKPOINT)) {
return null;
}
ASTBooleanLiteral loadCheckpoint = (ASTBooleanLiteral) getSimpleConfigurationValue(LOAD_CHECKPOINT);
return loadCheckpoint.getValue();
}
public Integer getCheckpointPeriod() {
if (!configurationContainsKey(CHECKPOINT_PERIOD)) {
return null;
}
ASTSignedIntLiteral checkpointPeriod = (ASTSignedIntLiteral) getSimpleConfigurationValue(CHECKPOINT_PERIOD);
return checkpointPeriod.getValue();
}
public Integer getLogPeriod() {
if (!configurationContainsKey(LOG_PERIOD)) {
return null;
}
ASTSignedIntLiteral logPeriod = (ASTSignedIntLiteral) getSimpleConfigurationValue(LOG_PERIOD);
return logPeriod.getValue();
}
public Boolean getLoadPretrained() {
if (!configurationContainsKey(LOAD_PRETRAINED)) {
return null;
}
ASTBooleanLiteral loadPretrained = (ASTBooleanLiteral) getSimpleConfigurationValue(LOAD_PRETRAINED);
return loadPretrained.getValue();
}
// COMPARE WITH CNNTRAIN IMPLEMENTATION IN GluonConfigurationData
public Boolean getPreprocessor() {
return configurationContainsKey(PREPROCESSING_NAME);
}
public Boolean getNormalize() {
if (!configurationContainsKey(NORMALIZE)) {
return null;
}
ASTBooleanLiteral normalize = (ASTBooleanLiteral) getSimpleConfigurationValue(NORMALIZE);
return normalize.getValue();
}
public Boolean getShuffleData() {
if (!configurationContainsKey(SHUFFLE_DATA)) {
return null;
}
ASTBooleanLiteral shuffleData = (ASTBooleanLiteral) getSimpleConfigurationValue(SHUFFLE_DATA);
return shuffleData.getValue();
}
public Double getClipGlobalGradNorm() {
if (!configurationContainsKey(CLIP_GLOBAL_GRAD_NORM)) {
return null;
}
ASTSignedDoubleLiteral clipGlobalGradNorm = (ASTSignedDoubleLiteral) getSimpleConfigurationValue(CLIP_GLOBAL_GRAD_NORM);
return clipGlobalGradNorm.getValue();
}
public Boolean getUseTeacherForcing() {
if (!configurationContainsKey(USE_TEACHER_FORCING)) {
return null;
}
ASTBooleanLiteral useTeacherForcing = (ASTBooleanLiteral) getSimpleConfigurationValue(USE_TEACHER_FORCING);
return useTeacherForcing.getValue();
}
public Boolean getSaveAttentionImage() {
if (!configurationContainsKey(SAVE_ATTENTION_IMAGE)) {
return null;
}
ASTBooleanLiteral saveAttentionImage = (ASTBooleanLiteral) getSimpleConfigurationValue(SAVE_ATTENTION_IMAGE);
return saveAttentionImage.getValue();
}
public Boolean getEvalTrain() {
if (!configurationContainsKey(EVAL_TRAIN)) {
return null;
}
ASTBooleanLiteral evalTrain = (ASTBooleanLiteral) getSimpleConfigurationValue(EVAL_TRAIN);
return evalTrain.getValue();
}
public String getContext() {
if (!configurationContainsKey(CONTEXT)) {
return null;
}
ASTTypelessLiteral context = (ASTTypelessLiteral) getSimpleConfigurationValue(CONTEXT);
return context.getValue();
}
/*
* Optimizer is of type NestedConfigurationEntry, so we need following methods.
*
* 1. Method to check whether this configuration exists, see getOptimizer().
* 2. Method to get the name of the NestedConfigurationEntry, see getOptimizerName().
* 3. Method to get the nested parameters, see getOptimizerParameters().
*/
public Boolean getOptimizer() {
return configurationContainsKey(OPTIMIZER);
}
public String getOptimizerName() {
if (!getOptimizer()) {
return null;
}
String optimizerName = null;
SymbolKind symbolKind = configuration.configurationSymbolKind(OPTIMIZER);
if (SimpleConfigurationEntrySymbol.KIND.isSame(symbolKind)) {
ASTTypelessLiteral optimizer = (ASTTypelessLiteral) getSimpleConfigurationValue(OPTIMIZER);
optimizerName = optimizer.getValue();
} else if (NestedConfigurationEntrySymbol.KIND.isSame(symbolKind)) {
ASTNestedConfigurationEntry optimizer = getNestedConfigurationValue(OPTIMIZER);
ASTTypelessLiteral value = (ASTTypelessLiteral) optimizer.getValue();
optimizerName = value.getValue();
}
return optimizerName;
}
public Map<String, Object> getOptimizerParameters() {
if (!getOptimizer()) {
return null;
}
if (!NestedConfigurationEntrySymbol.KIND.isSame(configuration.configurationSymbolKind(OPTIMIZER))) {
return null;
}
ASTNestedConfigurationEntry optimizer = getNestedConfigurationValue(OPTIMIZER);
List<ASTConfigurationEntry> configurationEntries = optimizer.getConfigurationEntryList();
Map<String, Object> keyValues = Maps.newHashMap();
for (ASTConfigurationEntry astConfigurationEntry : configurationEntries) {
if (astConfigurationEntry.getClass().equals(ASTSimpleConfigurationEntry.class)) {
ASTSimpleConfigurationEntry entry = (ASTSimpleConfigurationEntry) astConfigurationEntry;
ASTSignedLiteral value = entry.getValue();
keyValues.put(entry.getName(), getValue(value));
}
}
return keyValues;
}
public Boolean getActorOptimizer() {
return configurationContainsKey(ACTOR_OPTIMIZER);
}
public String getActorOptimizerName() {
if (!getActorOptimizer()) {
return null;
}
String actorOptimizerName = null;
SymbolKind symbolKind = configuration.configurationSymbolKind(ACTOR_OPTIMIZER);
if (SimpleConfigurationEntrySymbol.KIND.isSame(symbolKind)) {
ASTTypelessLiteral actorOptimizer = (ASTTypelessLiteral) getSimpleConfigurationValue(ACTOR_OPTIMIZER);
actorOptimizerName = actorOptimizer.getValue();
} else if (NestedConfigurationEntrySymbol.KIND.isSame(symbolKind)) {
ASTNestedConfigurationEntry actorOptimizer = getNestedConfigurationValue(ACTOR_OPTIMIZER);
ASTTypelessLiteral value = (ASTTypelessLiteral) actorOptimizer.getValue();
actorOptimizerName = value.getValue();
}
return actorOptimizerName;
}
public Map<String, Object> getActorOptimizerParameters() {
if (!getActorOptimizer()) {
return null;
}
if (!NestedConfigurationEntrySymbol.KIND.isSame(configuration.configurationSymbolKind(ACTOR_OPTIMIZER))) {
return null;
}
ASTNestedConfigurationEntry actorOptimizer = getNestedConfigurationValue(ACTOR_OPTIMIZER);
List<ASTConfigurationEntry> configurationEntries = actorOptimizer.getConfigurationEntryList();
Map<String, Object> keyValues = Maps.newHashMap();
for (ASTConfigurationEntry astConfigurationEntry : configurationEntries) {
if (astConfigurationEntry.getClass().equals(ASTSimpleConfigurationEntry.class)) {
ASTSimpleConfigurationEntry entry = (ASTSimpleConfigurationEntry) astConfigurationEntry;
ASTSignedLiteral value = entry.getValue();
keyValues.put(entry.getName(), getValue(value));
}
}
return keyValues;
}
public Boolean getCriticOptimizer() {
return configurationContainsKey(CRITIC_OPTIMIZER);
}
public String getCriticOptimizerName() {
if (!getCriticOptimizer()) {
return null;
}
String criticOptimizerName = null;
SymbolKind symbolKind = configuration.configurationSymbolKind(CRITIC_OPTIMIZER);
if (SimpleConfigurationEntrySymbol.KIND.isSame(symbolKind)) {
ASTTypelessLiteral criticOptimizer = (ASTTypelessLiteral) getSimpleConfigurationValue(CRITIC_OPTIMIZER);
criticOptimizerName = criticOptimizer.getValue();
} else if (NestedConfigurationEntrySymbol.KIND.isSame(symbolKind)) {
ASTNestedConfigurationEntry criticOptimizer = getNestedConfigurationValue(CRITIC_OPTIMIZER);
ASTTypelessLiteral value = (ASTTypelessLiteral) criticOptimizer.getValue();