Skip to content

GitLab

C
CNNArchLang
Commit 766d34c4 authored by Julian Johannes Steinsberger-Dührßen's avatar Julian Johannes Steinsberger-Dührßen
Browse files
Options

Added replay for LargeMemory layer

parent 1558a321
Hide whitespace changes
Inline Side-by-side
Showing with 151 additions and 19 deletions
src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/ArchitectureSymbol.java
View file @ 766d34c4
......@@ -222,11 +222,40 @@ public class ArchitectureSymbol extends CommonScopeSpanningSymbol {
List<ArchitectureElementSymbol> currentReplaySubNetworkElements = new ArrayList<>();
for (ArchitectureElementSymbol element : elements){
if (element.getName().equals("ReplayMemory")) {
if (!currentReplaySubNetworkElements.isEmpty()){
replaySubNetworks.add(currentReplaySubNetworkElements);
if (AllPredefinedLayers.REPLAY_LAYER_NAMES.contains(element.getName())) {
boolean use_replay = false;
boolean use_local_adaption = false;
for (ArgumentSymbol arg : ((LayerSymbol)element).getArguments()){
if (arg.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME) && (boolean)arg.getRhs().getValue().get()){
use_replay = true;
break;
}else if (arg.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME) && (boolean)arg.getRhs().getValue().get()){
use_local_adaption = true;
break;
}
}
if (!use_replay && !use_local_adaption) {
for (ParameterSymbol param : ((LayerSymbol) element).getDeclaration().getParameters()) {
if (param.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME) &&
(boolean) param.getDefaultExpression().get().getValue().get()) {
use_replay = true;
break;
} else if (param.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME) &&
(boolean) param.getDefaultExpression().get().getValue().get()) {
use_local_adaption = true;
break;
}
}
}
if (use_replay || use_local_adaption){
if (!currentReplaySubNetworkElements.isEmpty()){
replaySubNetworks.add(currentReplaySubNetworkElements);
}
currentReplaySubNetworkElements = new ArrayList<>();
}
currentReplaySubNetworkElements = new ArrayList<>();
}
currentReplaySubNetworkElements.add(element);
}
......
src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/Constraints.java
View file @ 766d34c4
......@@ -259,6 +259,26 @@ public enum Constraints {
+ AllPredefinedLayers.MEMORY_ACTIVATION_SOFTSIGN;
}
},
DIST_MEASURE_TYPE {
@Override
public boolean isValid(ArchSimpleExpressionSymbol exp) {
Optional<String> optString= exp.getStringValue();
if (optString.isPresent()){
if (optString.get().equals(AllPredefinedLayers.L2)
|| optString.get().equals(AllPredefinedLayers.INNER_PROD)){
return true;
}
}
return false;
}
@Override
protected String msgString() {
return AllPredefinedLayers.L2 + " or "
+ AllPredefinedLayers.INNER_PROD + "or"
+ AllPredefinedLayers.RANDOM;
}
},
NULLABLE_AXIS {
@Override
public boolean isValid(ArchSimpleExpressionSymbol exp) {
......
src/main/java/de/monticore/lang/monticar/cnnarch/predefined/AllPredefinedLayers.java
View file @ 766d34c4
......@@ -13,6 +13,7 @@ import de.monticore.lang.monticar.cnnarch._symboltable.UnrollDeclarationSymbol;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
public class AllPredefinedLayers {
......@@ -53,8 +54,13 @@ public class AllPredefinedLayers {
public static final String SWAPAXES_NAME = "SwapAxes";
public static final String BROADCAST_ADD_NAME = "BroadcastAdd";
public static final String RESHAPE_NAME = "Reshape";
//replay layers
public static final String MEMORY_NAME = "Memory";
public static final String REPLAY_MEMORY_NAME = "ReplayMemory";
public static final List<String> REPLAY_LAYER_NAMES = new ArrayList<String>(Arrays.asList(MEMORY_NAME,
REPLAY_MEMORY_NAME));
//predefined argument names
public static final String KERNEL_NAME = "kernel";
......@@ -90,6 +96,19 @@ public class AllPredefinedLayers {
public static final String SHAPE_NAME = "shape";
public static final String RNN_DROPOUT_NAME = "dropout";
//shared parameters replay layers
public static final String USE_REPLAY_NAME = "useReplay";
public static final String REPLAY_INTERVAL_NAME = "replayInterval";
public static final String REPLAY_BATCH_SIZE_NAME = "replayBatchSize";
public static final String REPLAY_STEPS_NAME = "replaySteps";
public static final String REPLAY_GRADIENT_STEPS_NAME = "replayGradientSteps";
public static final String REPLAY_MEMORY_STORE_PROB_NAME = "replayMemoryStoreProb";
public static final String REPLAY_MEMORY_STORE_DIST_MEASURE_NAME = "replayMemoryStoreDistMeasure";
public static final String USE_LOCAL_ADAPTION_NAME = "useLocalAdaption";
public static final String LOCAL_ADAPTION_K_NAME = "localAdaptionK";
public static final String LOCAL_ADAPTION_GRADIENT_STEPS_NAME = "localAdaptionGradientSteps";
public static final String LOCAL_ADAPTION_MEMORY_STORE_DIST_MEASURE_NAME = "localAdaptionMemoryStoreDistMeasure";
//parameters for memory layer
public static final String SUB_KEY_SIZE_NAME = "subKeySize";
public static final String QUERY_SIZE_NAME = "querySize";
......@@ -98,15 +117,9 @@ public class AllPredefinedLayers {
public static final String NUM_HEADS_NAME = "numHeads";
public static final String VALUE_SHAPE_NAME = "valueShape";
//parameters for replay memory layer
public static final String REPLAY_INTERVAL_NAME = "replayInterval";
public static final String REPLAY_BATCH_SIZE_NAME = "replayBatchSize";
public static final String REPLAY_STEPS_NAME = "replaySteps";
public static final String REPLAY_GRADIENT_STEPS_TRAINING_NAME = "replayGradientStepsTraining";
public static final String STORE_PROB_NAME = "storeProb";
public static final String MAX_STORED_SAMPLES_NAME = "maxStoredSamples";
public static final String REPLAY_K_NAME = "replayK";
public static final String REPLAY_GRADIENT_STEPS_PREDICTION_NAME = "replayGradientStepsPrediction";
public static final String QUERY_NET_DIR_NAME = "queryNetDir";
public static final String QUERY_NET_PREFIX_NAME = "queryNetPrefix";
......@@ -116,6 +129,9 @@ public class AllPredefinedLayers {
public static final String PADDING_NO_LOSS = "no_loss";
public static final String POOL_MAX = "max";
public static final String POOL_AVG = "avg";
public static final String L2 = "l2";
public static final String INNER_PROD = "inner_prod";
public static final String RANDOM = "random";
//possible activation values for the querry network in the memory layer
public static final String MEMORY_ACTIVATION_LINEAR = "linear";
......
src/main/java/de/monticore/lang/monticar/cnnarch/predefined/Memory.java
View file @ 766d34c4
......@@ -60,6 +60,59 @@ public class Memory extends PredefinedLayerDeclaration {
public static Memory create(){
Memory declaration = new Memory();
List<ParameterSymbol> parameters = new ArrayList<>(Arrays.asList(
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.USE_REPLAY_NAME)
.constraints(Constraints.BOOLEAN)
.defaultValue(false)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_INTERVAL_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_BATCH_SIZE_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
.defaultValue(-1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_STEPS_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_MEMORY_STORE_PROB_NAME)
.constraints(Constraints.NUMBER, Constraints.BETWEEN_ZERO_AND_ONE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME)
.constraints(Constraints.BOOLEAN, Constraints.POSITIVE)
.defaultValue(false)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.LOCAL_ADAPTION_K_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.LOCAL_ADAPTION_GRADIENT_STEPS_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_MEMORY_STORE_DIST_MEASURE_NAME)
.constraints(Constraints.DIST_MEASURE_TYPE)
.defaultValue(AllPredefinedLayers.INNER_PROD)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.LOCAL_ADAPTION_MEMORY_STORE_DIST_MEASURE_NAME)
.constraints(Constraints.DIST_MEASURE_TYPE)
.defaultValue(AllPredefinedLayers.INNER_PROD)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.SUB_KEY_SIZE_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
......
src/main/java/de/monticore/lang/monticar/cnnarch/predefined/ReplayMemory.java
View file @ 766d34c4
......@@ -42,6 +42,11 @@ public class ReplayMemory extends PredefinedLayerDeclaration {
public static ReplayMemory create(){
ReplayMemory declaration = new ReplayMemory();
List<ParameterSymbol> parameters = new ArrayList<>(Arrays.asList(
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.USE_REPLAY_NAME)
.constraints(Constraints.BOOLEAN)
.defaultValue(true)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_INTERVAL_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
......@@ -54,30 +59,39 @@ public class ReplayMemory extends PredefinedLayerDeclaration {
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_STEPS_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue("linear")
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_TRAINING_NAME)
.name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.STORE_PROB_NAME)
.name(AllPredefinedLayers.REPLAY_MEMORY_STORE_PROB_NAME)
.constraints(Constraints.NUMBER, Constraints.BETWEEN_ZERO_AND_ONE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.MAX_STORED_SAMPLES_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
.defaultValue(-1)
.name(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME)
.constraints(Constraints.BOOLEAN, Constraints.POSITIVE)
.defaultValue(true)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_K_NAME)
.name(AllPredefinedLayers.LOCAL_ADAPTION_GRADIENT_STEPS_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue(1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_PREDICTION_NAME)
.name(AllPredefinedLayers.REPLAY_MEMORY_STORE_DIST_MEASURE_NAME)
.constraints(Constraints.DIST_MEASURE_TYPE)
.defaultValue(AllPredefinedLayers.L2)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.MAX_STORED_SAMPLES_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
.defaultValue(-1)
.build(),
new ParameterSymbol.Builder()
.name(AllPredefinedLayers.LOCAL_ADAPTION_K_NAME)
.constraints(Constraints.INTEGER, Constraints.POSITIVE)
.defaultValue(1)
.build(),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment