Added replay for LargeMemory layer

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/ArchitectureSymbol.java

@@ -222,11 +222,40 @@ public class ArchitectureSymbol extends CommonScopeSpanningSymbol {
             List<ArchitectureElementSymbol> currentReplaySubNetworkElements = new ArrayList<>();
 
             for (ArchitectureElementSymbol element : elements){
-                if (element.getName().equals("ReplayMemory")) {
-                    if (!currentReplaySubNetworkElements.isEmpty()){
-                        replaySubNetworks.add(currentReplaySubNetworkElements);
+                if (AllPredefinedLayers.REPLAY_LAYER_NAMES.contains(element.getName())) {
+                    boolean use_replay = false;
+                    boolean use_local_adaption = false;
+
+                    for (ArgumentSymbol arg : ((LayerSymbol)element).getArguments()){
+                        if (arg.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME) && (boolean)arg.getRhs().getValue().get()){
+                            use_replay = true;
+                            break;
+                        }else if (arg.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME) && (boolean)arg.getRhs().getValue().get()){
+                            use_local_adaption = true;
+                            break;
+                        }
+                    }
+
+                    if (!use_replay && !use_local_adaption) {
+                        for (ParameterSymbol param : ((LayerSymbol) element).getDeclaration().getParameters()) {
+                            if (param.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME) &&
+                                    (boolean) param.getDefaultExpression().get().getValue().get()) {
+                                use_replay = true;
+                                break;
+                            } else if (param.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME) &&
+                                    (boolean) param.getDefaultExpression().get().getValue().get()) {
+                                use_local_adaption = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (use_replay || use_local_adaption){
+                        if (!currentReplaySubNetworkElements.isEmpty()){
+                            replaySubNetworks.add(currentReplaySubNetworkElements);
+                        }
+                        currentReplaySubNetworkElements = new ArrayList<>();
                     }
-                    currentReplaySubNetworkElements = new ArrayList<>();
                 }
                 currentReplaySubNetworkElements.add(element);
             }

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/Constraints.java

@@ -259,6 +259,26 @@ public enum Constraints {
                     + AllPredefinedLayers.MEMORY_ACTIVATION_SOFTSIGN;
         }
     },
+    DIST_MEASURE_TYPE {
+        @Override
+        public boolean isValid(ArchSimpleExpressionSymbol exp) {
+            Optional<String> optString= exp.getStringValue();
+            if (optString.isPresent()){
+                if (optString.get().equals(AllPredefinedLayers.L2)
+                        || optString.get().equals(AllPredefinedLayers.INNER_PROD)){
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        @Override
+        protected String msgString() {
+            return AllPredefinedLayers.L2 + " or "
+                    + AllPredefinedLayers.INNER_PROD + "or"
+                    + AllPredefinedLayers.RANDOM;
+        }
+    },
     NULLABLE_AXIS {
         @Override
         public boolean isValid(ArchSimpleExpressionSymbol exp) {

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/AllPredefinedLayers.java

@@ -13,6 +13,7 @@ import de.monticore.lang.monticar.cnnarch._symboltable.UnrollDeclarationSymbol;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.ArrayList;
 
 public class AllPredefinedLayers {
 
@@ -53,8 +54,13 @@ public class AllPredefinedLayers {
     public static final String SWAPAXES_NAME = "SwapAxes";
     public static final String BROADCAST_ADD_NAME = "BroadcastAdd";
     public static final String RESHAPE_NAME = "Reshape";
+
+    //replay layers
     public static final String MEMORY_NAME = "Memory";
     public static final String REPLAY_MEMORY_NAME = "ReplayMemory";
+    public static final List<String> REPLAY_LAYER_NAMES = new ArrayList<String>(Arrays.asList(MEMORY_NAME,
+                                                                                              REPLAY_MEMORY_NAME));
+
 
     //predefined argument names
     public static final String KERNEL_NAME = "kernel";
@@ -90,6 +96,19 @@ public class AllPredefinedLayers {
     public static final String SHAPE_NAME = "shape";
     public static final String RNN_DROPOUT_NAME = "dropout";
 
+    //shared parameters replay layers
+    public static final String USE_REPLAY_NAME = "useReplay";
+    public static final String REPLAY_INTERVAL_NAME = "replayInterval";
+    public static final String REPLAY_BATCH_SIZE_NAME = "replayBatchSize";
+    public static final String REPLAY_STEPS_NAME = "replaySteps";
+    public static final String REPLAY_GRADIENT_STEPS_NAME = "replayGradientSteps";
+    public static final String REPLAY_MEMORY_STORE_PROB_NAME = "replayMemoryStoreProb";
+    public static final String REPLAY_MEMORY_STORE_DIST_MEASURE_NAME = "replayMemoryStoreDistMeasure";
+    public static final String USE_LOCAL_ADAPTION_NAME = "useLocalAdaption";
+    public static final String LOCAL_ADAPTION_K_NAME = "localAdaptionK";
+    public static final String LOCAL_ADAPTION_GRADIENT_STEPS_NAME = "localAdaptionGradientSteps";
+    public static final String LOCAL_ADAPTION_MEMORY_STORE_DIST_MEASURE_NAME = "localAdaptionMemoryStoreDistMeasure";
+
 	//parameters for memory layer
     public static final String SUB_KEY_SIZE_NAME = "subKeySize";
     public static final String QUERY_SIZE_NAME = "querySize";
@@ -98,15 +117,9 @@ public class AllPredefinedLayers {
 	public static final String NUM_HEADS_NAME = "numHeads";
 	public static final String VALUE_SHAPE_NAME = "valueShape";
 
+
     //parameters for replay memory layer
-    public static final String REPLAY_INTERVAL_NAME = "replayInterval";
-    public static final String REPLAY_BATCH_SIZE_NAME = "replayBatchSize";
-	public static final String REPLAY_STEPS_NAME = "replaySteps";
-    public static final String REPLAY_GRADIENT_STEPS_TRAINING_NAME = "replayGradientStepsTraining";
-	public static final String STORE_PROB_NAME = "storeProb";
 	public static final String MAX_STORED_SAMPLES_NAME = "maxStoredSamples";
-    public static final String REPLAY_K_NAME = "replayK";
-    public static final String REPLAY_GRADIENT_STEPS_PREDICTION_NAME = "replayGradientStepsPrediction";
 	public static final String QUERY_NET_DIR_NAME = "queryNetDir";
 	public static final String QUERY_NET_PREFIX_NAME = "queryNetPrefix";
 
@@ -116,6 +129,9 @@ public class AllPredefinedLayers {
     public static final String PADDING_NO_LOSS = "no_loss";
     public static final String POOL_MAX = "max";
     public static final String POOL_AVG = "avg";
+    public static final String L2 = "l2";
+    public static final String INNER_PROD = "inner_prod";
+    public static final String RANDOM = "random";
 	
 	//possible activation values for the querry network in the memory layer
 	public static final String MEMORY_ACTIVATION_LINEAR = "linear";

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/Memory.java

@@ -60,6 +60,59 @@ public class Memory extends PredefinedLayerDeclaration {
     public static Memory create(){
         Memory declaration = new Memory();
         List<ParameterSymbol> parameters = new ArrayList<>(Arrays.asList(
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.USE_REPLAY_NAME)
+                        .constraints(Constraints.BOOLEAN)
+                        .defaultValue(false)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.REPLAY_INTERVAL_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.REPLAY_BATCH_SIZE_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
+                        .defaultValue(-1)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.REPLAY_STEPS_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .defaultValue(1)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.REPLAY_MEMORY_STORE_PROB_NAME)
+                        .constraints(Constraints.NUMBER, Constraints.BETWEEN_ZERO_AND_ONE)
+                        .defaultValue(1)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME)
+                        .constraints(Constraints.BOOLEAN, Constraints.POSITIVE)
+                        .defaultValue(false)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.LOCAL_ADAPTION_K_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .defaultValue(1)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.LOCAL_ADAPTION_GRADIENT_STEPS_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .defaultValue(1)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.REPLAY_MEMORY_STORE_DIST_MEASURE_NAME)
+                        .constraints(Constraints.DIST_MEASURE_TYPE)
+                        .defaultValue(AllPredefinedLayers.INNER_PROD)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.LOCAL_ADAPTION_MEMORY_STORE_DIST_MEASURE_NAME)
+                        .constraints(Constraints.DIST_MEASURE_TYPE)
+                        .defaultValue(AllPredefinedLayers.INNER_PROD)
+                        .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.SUB_KEY_SIZE_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/ReplayMemory.java

@@ -42,6 +42,11 @@ public class ReplayMemory extends PredefinedLayerDeclaration {
     public static ReplayMemory create(){
         ReplayMemory declaration = new ReplayMemory();
         List<ParameterSymbol> parameters = new ArrayList<>(Arrays.asList(
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.USE_REPLAY_NAME)
+                        .constraints(Constraints.BOOLEAN)
+                        .defaultValue(true)
+                        .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.REPLAY_INTERVAL_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
@@ -54,30 +59,39 @@ public class ReplayMemory extends PredefinedLayerDeclaration {
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.REPLAY_STEPS_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
-                        .defaultValue("linear")
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_TRAINING_NAME)
+                        .name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
                         .defaultValue(1)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.STORE_PROB_NAME)
+                        .name(AllPredefinedLayers.REPLAY_MEMORY_STORE_PROB_NAME)
                         .constraints(Constraints.NUMBER, Constraints.BETWEEN_ZERO_AND_ONE)
                         .defaultValue(1)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.MAX_STORED_SAMPLES_NAME)
-                        .constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
-                        .defaultValue(-1)
+                        .name(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME)
+                        .constraints(Constraints.BOOLEAN, Constraints.POSITIVE)
+                        .defaultValue(true)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.REPLAY_K_NAME)
+                        .name(AllPredefinedLayers.LOCAL_ADAPTION_GRADIENT_STEPS_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
                         .defaultValue(1)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.REPLAY_GRADIENT_STEPS_PREDICTION_NAME)
+                        .name(AllPredefinedLayers.REPLAY_MEMORY_STORE_DIST_MEASURE_NAME)
+                        .constraints(Constraints.DIST_MEASURE_TYPE)
+                        .defaultValue(AllPredefinedLayers.L2)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.MAX_STORED_SAMPLES_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
+                        .defaultValue(-1)
+                        .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.LOCAL_ADAPTION_K_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
                         .defaultValue(1)
                         .build(),