Merge branch 'develop' into 'master'

Develop

See merge request !39

README.md

@@ -606,6 +606,8 @@ All predefined methods start with a capital letter and all constructed methods h
     
 * **EpisodicMemory(replayMemoryStoreProb=1, maxStoredSamples=-1, memoryReplacementStrategy="replace_oldest", useReplay=true, replayInterval, replayBatchSize=-1, replaySteps, replayGradientSteps=1, useLocalAdaption=true, localAdaptionGradientSteps=1, localAdaptionK=1, queryNetDir=-1, queryNetPrefix=-1, queryNetNumInputs=1)**
 
+!!! Important: Needs mxnet version 1.7.0, see installation instructions at: [EMADL2CPP](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP)!!!
+
   Episodic Memory as described in [1], although we implemented it as a layer which can also be used inside of a network. Works with multiple inputs. Not learned. Inputs are stored and are repleayed on part of the network following this layer. Local adaption retrives samples to the sample for which inference should be done from memory and performs finetuning learning with these on the part of the network following this layer.
 
   * **replayMemoryStoreProb** ( 0 <= integer <= 1, optional, default=1): Probability with which a sample seen during training will be stored in memory (per actual sample, not batch).
@@ -639,6 +641,8 @@ All predefined methods start with a capital letter and all constructed methods h
 
 * **DotProductSelfAttention(scaleFactor=-1, numHeads=1, dimKeys=-1, dimValues=-1, useProjBias=true, useMask=false)**
 
+!!! Important: Needs mxnet version 1.7.0, see installation instructions at: [EMADL2CPP](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP)!!!
+
   Calculates the DotProductSelfAttention. As described in [3]. Takes three inputs: queries, keys, values and optionally a mask for masked Self Attention as used in Bert.
   
   * **scaleFactor** (integer > 0 or -1, optional, default=-1): factor to scale the score of the dot porduct of querries and keys, for -1 this is sqrt(dimKeys).
@@ -677,6 +681,6 @@ All predefined methods start with a capital letter and all constructed methods h
 
 ## Refrences
 
-  [1] Cyprien De Masson, Sebastian Ruder, Lingpeng Kong and Dani Yogatama, "Episodic Memory in Lifelong Language Learning", Proc. NeurIPS, 2019
-  [2] Guillaume Lample, Alexandre Sablayrolles, Marc’Aurelio Ranzato, Ludovic Denoyer and Hervé Jégou, "Large memory layers with product keys", Proc. NeurIPS, Dec. 2019.
+  [1] Cyprien De Masson, Sebastian Ruder, Lingpeng Kong and Dani Yogatama, "Episodic Memory in Lifelong Language Learning", Proc. NeurIPS, 2019 \
+  [2] Guillaume Lample, Alexandre Sablayrolles, Marc’Aurelio Ranzato, Ludovic Denoyer and Hervé Jégou, "Large memory layers with product keys", Proc. NeurIPS, Dec. 2019. \
   [3] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Lukasz Kaiser and Illia Polosukhin, "Attention is all you need", Proc. NeurIPS, 2017

src/main/java/de/monticore/lang/monticar/cnnarch/_cocos/CheckLargeMemoryLayer.java

@@ -32,9 +32,9 @@ public class CheckLargeMemoryLayer extends CNNArchSymbolCoCo {
         Integer k = new Integer(0);
 
         for (ArgumentSymbol arg : arguments) {
-            if (arg.getName().equals("subKeySize")) {
+            if (arg.getName().equals("subKeySize") && arg.getRhs().getIntValue().isPresent()) {
                 subKeySize = arg.getRhs().getIntValue().get();
-            } else if (arg.getName().equals("k")) {
+            } else if (arg.getName().equals("k") && arg.getRhs().getIntValue().isPresent()) {
                 k = arg.getRhs().getIntValue().get();
             }
         }

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/ArchExpressionSymbol.java

@@ -229,6 +229,30 @@ abstract public class ArchExpressionSymbol extends CommonSymbol {
         return Optional.empty();
     }
 
+    public Optional<List<Integer>> getIntOrIntTupleValues(){
+        Optional<List<Object>> optValue = getTupleValues();
+        if (optValue.isPresent()){
+            List<Integer> list = new ArrayList<>();
+            for (Object value : optValue.get()) {
+                if (value instanceof Integer){
+                    list.add((Integer) value);
+                }
+                else {
+                    return Optional.empty();
+                }
+            }
+            return Optional.of(list);
+        }else{
+            List<Integer> list = new ArrayList<>();
+            Optional<Integer> optValueInt = getIntValue();
+            if (optValueInt.isPresent()){
+                list.add(optValueInt.get());
+                return Optional.of(list);
+            }
+        }
+        return Optional.empty();
+    }
+
     public Optional<List<Double>> getDoubleTupleValues() {
         Optional<List<Object>> optValue = getTupleValues();
         if (optValue.isPresent()){

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/ArchitectureSymbol.java

@@ -260,37 +260,53 @@ public class ArchitectureSymbol extends CommonScopeSpanningSymbol {
             List<ArchitectureElementSymbol> elementsNew = new ArrayList<>();
             List<List<ArchitectureElementSymbol>> episodicSubNetworks = new ArrayList<>(new ArrayList<>());
             List<ArchitectureElementSymbol> currentEpisodicSubNetworkElements = new ArrayList<>();
+            boolean anyEpisodicLocalAdaptation = false;
 
             for (ArchitectureElementSymbol element : elements){
                 if (AllPredefinedLayers.EPISODIC_REPLAY_LAYER_NAMES.contains(element.getName())) {
                     boolean use_replay = false;
-                    boolean use_local_adaption = false;
+                    boolean use_local_adaptation = false;
+                    boolean use_replay_specified = false;
+                    boolean use_local_adaptation_specified = false;
 
                     for (ArgumentSymbol arg : ((LayerSymbol)element).getArguments()){
-                        if (arg.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME) && (boolean)arg.getRhs().getValue().get()){
-                            use_replay = true;
-                            break;
-                        }else if (arg.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME) && (boolean)arg.getRhs().getValue().get()){
-                            use_local_adaption = true;
-                            break;
+                        if (arg.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME)){
+                                use_replay_specified = true;
+                                if ((boolean)arg.getRhs().getValue().get()) {
+                                    use_replay = true;
+                                    break;
+                                }
+                        }else if (arg.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTATION_NAME)){
+                            use_local_adaptation_specified = true;
+                            if ((boolean)arg.getRhs().getValue().get()) {
+                                use_local_adaptation = true;
+                                anyEpisodicLocalAdaptation = true;
+                                break;
+                            }
                         }
                     }
 
-                    if (!use_replay && !use_local_adaption) {
+                    if (!use_replay_specified) {
                         for (ParameterSymbol param : ((LayerSymbol) element).getDeclaration().getParameters()) {
                             if (param.getName().equals(AllPredefinedLayers.USE_REPLAY_NAME) &&
                                     (boolean) param.getDefaultExpression().get().getValue().get()) {
                                 use_replay = true;
                                 break;
-                            } else if (param.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME) &&
+                            }
+                        }
+                    }
+                    if (!use_local_adaptation_specified) {
+                        for (ParameterSymbol param : ((LayerSymbol) element).getDeclaration().getParameters()) {
+                            if (param.getName().equals(AllPredefinedLayers.USE_LOCAL_ADAPTATION_NAME) &&
                                     (boolean) param.getDefaultExpression().get().getValue().get()) {
-                                use_local_adaption = true;
+                                use_local_adaptation = true;
+                                anyEpisodicLocalAdaptation = true;
                                 break;
                             }
                         }
                     }
 
-                    if (use_replay || use_local_adaption){
+                    if (use_replay || use_local_adaptation){
                         if (!currentEpisodicSubNetworkElements.isEmpty()){
                             episodicSubNetworks.add(currentEpisodicSubNetworkElements);
                         }
@@ -303,6 +319,7 @@ public class ArchitectureSymbol extends CommonScopeSpanningSymbol {
                 episodicSubNetworks.add(currentEpisodicSubNetworkElements);
             }
                 networkInstruction.getBody().setEpisodicSubNetworks(episodicSubNetworks);
+                networkInstruction.getBody().setAnyEpisodicLocalAdaptation(anyEpisodicLocalAdaptation);
         }
     }
 }

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/Constraints.java

@@ -269,26 +269,6 @@ public enum Constraints {
                     + AllPredefinedLayers.MEMORY_ACTIVATION_SOFTSIGN;
         }
     },
-    DIST_MEASURE_TYPE {
-        @Override
-        public boolean isValid(ArchSimpleExpressionSymbol exp) {
-            Optional<String> optString= exp.getStringValue();
-            if (optString.isPresent()){
-                if (optString.get().equals(AllPredefinedLayers.L2)
-                        || optString.get().equals(AllPredefinedLayers.INNER_PROD)){
-                    return true;
-                }
-            }
-            return false;
-        }
-
-        @Override
-        protected String msgString() {
-            return AllPredefinedLayers.L2 + " or "
-                    + AllPredefinedLayers.INNER_PROD + "or"
-                    + AllPredefinedLayers.RANDOM;
-        }
-    },
     MEMORY_REPLACEMENT_STRATEGY_TYPE {
         @Override
         public boolean isValid(ArchSimpleExpressionSymbol exp) {

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/LayerSymbol.java

@@ -283,6 +283,10 @@ public class LayerSymbol extends ArchitectureElementSymbol {
         return getTValue(parameterName, ArchExpressionSymbol::getIntTupleValues);
     }
 
+    public Optional<List<Integer>> getIntOrIntTupleValues(String parameterName){
+        return getTValue(parameterName, ArchExpressionSymbol::getIntOrIntTupleValues);
+    }
+
     public Optional<Boolean> getBooleanValue(String parameterName){
         return getTValue(parameterName, ArchExpressionSymbol::getBooleanValue);
     }

src/main/java/de/monticore/lang/monticar/cnnarch/_symboltable/SerialCompositeElementSymbol.java

@@ -13,6 +13,7 @@ import java.util.*;
 public class SerialCompositeElementSymbol extends CompositeElementSymbol {
 
     protected List<List<ArchitectureElementSymbol>> episodicSubNetworks = new ArrayList<>(new ArrayList<>());
+    protected boolean anyEpisodicLocalAdaptation = false;
 
     protected void setElements(List<ArchitectureElementSymbol> elements) {
         ArchitectureElementSymbol previous = null;
@@ -59,6 +60,10 @@ public class SerialCompositeElementSymbol extends CompositeElementSymbol {
     public List<List<ArchitectureElementSymbol>> getEpisodicSubNetworks() {
         return episodicSubNetworks;
     }
+    
+    protected void setAnyEpisodicLocalAdaptation(boolean value) { anyEpisodicLocalAdaptation = value; }
+
+    public boolean getAnyEpisodicLocalAdaptation() { return anyEpisodicLocalAdaptation; }
 
     @Override
     public void setInputElement(ArchitectureElementSymbol inputElement) {

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/AllPredefinedLayers.java

@@ -31,6 +31,7 @@ public class AllPredefinedLayers {
     public static final String GLOBAL_POOLING_NAME = "GlobalPooling";
     public static final String LRN_NAME = "Lrn";
     public static final String BATCHNORM_NAME = "BatchNorm";
+    public static final String LAYERNORM_NAME = "LayerNorm";
     public static final String SPLIT_NAME = "Split";
     public static final String GET_NAME = "Get";
     public static final String ADD_NAME = "Add";
@@ -116,9 +117,15 @@ public class AllPredefinedLayers {
     public static final String REPLAY_BATCH_SIZE_NAME = "replayBatchSize";
     public static final String REPLAY_STEPS_NAME = "replaySteps";
     public static final String REPLAY_GRADIENT_STEPS_NAME = "replayGradientSteps";
-    public static final String USE_LOCAL_ADAPTION_NAME = "useLocalAdaption";
-    public static final String LOCAL_ADAPTION_K_NAME = "localAdaptionK";
-    public static final String LOCAL_ADAPTION_GRADIENT_STEPS_NAME = "localAdaptionGradientSteps";
+    public static final String USE_LOCAL_ADAPTATION_NAME = "useLocalAdaptation";
+    public static final String LOCAL_ADAPTATION_K_NAME = "localAdaptationK";
+    public static final String LOCAL_ADAPTATION_GRADIENT_STEPS_NAME = "localAdaptationGradientSteps";
+    public static final String MAX_STORED_SAMPLES_NAME = "maxStoredSamples";
+    public static final String MEMORY_REPLACEMENT_STRATEGY_NAME = "memoryReplacementStrategy";
+    public static final String MEMORY_STORE_PROB_NAME = "memoryStoreProb";
+    public static final String QUERY_NET_DIR_NAME = "queryNetDir";
+    public static final String QUERY_NET_PREFIX_NAME = "queryNetPrefix";
+    public static final String QUERY_NET_NUM_INPUTS_NAME = "queryNetNumInputs";
 
 	//parameters for large memory layer
     public static final String SUB_KEY_SIZE_NAME = "subKeySize";
@@ -126,16 +133,7 @@ public class AllPredefinedLayers {
 	public static final String QUERY_ACT_NAME = "queryAct";
     public static final String K_NAME = "k";
 	public static final String NUM_HEADS_NAME = "numHeads";
-    public static final String STORE_DIST_MEASURE_NAME = "storeDistMeasure";
 	public static final String VALUES_DIM_NAME = "valuesDim";
-    public static final String MEMORY_REPLACEMENT_STRATEGY_NAME = "memoryReplacementStrategy";
-
-    //parameters for episodic memory layer
-	public static final String MAX_STORED_SAMPLES_NAME = "maxStoredSamples";
-    public static final String REPLAY_MEMORY_STORE_PROB_NAME = "replayMemoryStoreProb";
-	public static final String QUERY_NET_DIR_NAME = "queryNetDir";
-	public static final String QUERY_NET_PREFIX_NAME = "queryNetPrefix";
-    public static final String QUERY_NET_NUM_INPUTS_NAME = "queryNetNumInputs";
 
     //possible String values
     public static final String PADDING_VALID = "valid";
@@ -143,8 +141,6 @@ public class AllPredefinedLayers {
     public static final String PADDING_NO_LOSS = "no_loss";
     public static final String POOL_MAX = "max";
     public static final String POOL_AVG = "avg";
-    public static final String L2 = "l2";
-    public static final String INNER_PROD = "inner_prod";
     public static final String RANDOM = "random";
     public static final String REPLACE_OLDEST = "replace_oldest";
     public static final String NO_REPLACEMENT = "no_replacement";
@@ -174,6 +170,7 @@ public class AllPredefinedLayers {
                 GlobalPooling.create(),
                 Lrn.create(),
                 BatchNorm.create(),
+                LayerNorm.create(),
                 Split.create(),
                 Get.create(),
                 Add.create(),

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/DotProductSelfAttention.java

@@ -50,7 +50,7 @@ public class DotProductSelfAttention extends PredefinedLayerDeclaration {
         List<ParameterSymbol> parameters = new ArrayList<>(Arrays.asList(
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.SCALE_FACTOR_NAME)
-                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .constraints(Constraints.POSITIVE_OR_MINUS_ONE)
                         .defaultValue(-1)
                         .build(),
                 new ParameterSymbol.Builder()
@@ -60,12 +60,12 @@ public class DotProductSelfAttention extends PredefinedLayerDeclaration {
                         .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.DIM_KEYS_NAME)
-                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
                         .defaultValue(-1)
                         .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.DIM_VALUES_NAME)
-                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
                         .defaultValue(-1)
                         .build(),
                 new ParameterSymbol.Builder()

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/EpisodicMemory.java

@@ -74,29 +74,34 @@ public class EpisodicMemory extends PredefinedLayerDeclaration {
                         .defaultValue(1)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.REPLAY_MEMORY_STORE_PROB_NAME)
-                        .constraints(Constraints.NUMBER, Constraints.BETWEEN_ZERO_AND_ONE)
-                        .defaultValue(1)
+                        .name(AllPredefinedLayers.USE_LOCAL_ADAPTATION_NAME)
+                        .constraints(Constraints.BOOLEAN)
+                        .defaultValue(true)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.USE_LOCAL_ADAPTION_NAME)
-                        .constraints(Constraints.BOOLEAN, Constraints.POSITIVE)
-                        .defaultValue(true)
+                        .name(AllPredefinedLayers.LOCAL_ADAPTATION_GRADIENT_STEPS_NAME)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
+                        .defaultValue(1)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.LOCAL_ADAPTION_GRADIENT_STEPS_NAME)
+                        .name(AllPredefinedLayers.LOCAL_ADAPTATION_K_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
                         .defaultValue(1)
                         .build(),
+                new ParameterSymbol.Builder()
+                        .name(AllPredefinedLayers.MEMORY_STORE_PROB_NAME)
+                        .constraints(Constraints.NUMBER, Constraints.BETWEEN_ZERO_AND_ONE)
+                        .defaultValue(1)
+                        .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.MAX_STORED_SAMPLES_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE_OR_MINUS_ONE)
                         .defaultValue(-1)
                         .build(),
                 new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.LOCAL_ADAPTION_K_NAME)
-                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
-                        .defaultValue(1)
+                        .name(AllPredefinedLayers.MEMORY_REPLACEMENT_STRATEGY_NAME)
+                        .constraints(Constraints.MEMORY_REPLACEMENT_STRATEGY_TYPE)
+                        .defaultValue(AllPredefinedLayers.REPLACE_OLDEST)
                         .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.QUERY_NET_DIR_NAME)
@@ -109,11 +114,6 @@ public class EpisodicMemory extends PredefinedLayerDeclaration {
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.QUERY_NET_NUM_INPUTS_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)
-                        .build(),
-                new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.MEMORY_REPLACEMENT_STRATEGY_NAME)
-                        .constraints(Constraints.MEMORY_REPLACEMENT_STRATEGY_TYPE)
-                        .defaultValue(AllPredefinedLayers.REPLACE_OLDEST)
                         .build()));
         declaration.setParameters(parameters);
         return declaration;

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/LargeMemory.java

@@ -60,11 +60,6 @@ public class LargeMemory extends PredefinedLayerDeclaration {
     public static LargeMemory create(){
         LargeMemory declaration = new LargeMemory();
         List<ParameterSymbol> parameters = new ArrayList<>(Arrays.asList(
-                new ParameterSymbol.Builder()
-                        .name(AllPredefinedLayers.STORE_DIST_MEASURE_NAME)
-                        .constraints(Constraints.DIST_MEASURE_TYPE)
-                        .defaultValue(AllPredefinedLayers.INNER_PROD)
-                        .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.SUB_KEY_SIZE_NAME)
                         .constraints(Constraints.INTEGER, Constraints.POSITIVE)

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/LayerNorm.java

+/**
+ *
+ * (c) https://github.com/MontiCore/monticore
+ *
+ * The license generally applicable for this project
+ * can be found under https://github.com/MontiCore/monticore.
+ */
+/* (c) https://github.com/MontiCore/monticore */
+package de.monticore.lang.monticar.cnnarch.predefined;
+
+import de.monticore.lang.monticar.cnnarch._symboltable.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class LayerNorm extends PredefinedLayerDeclaration {
+
+    private LayerNorm() {
+        super(AllPredefinedLayers.LAYERNORM_NAME);
+    }
+
+    @Override
+    public List<ArchTypeSymbol> computeOutputTypes(List<ArchTypeSymbol> inputTypes, LayerSymbol layer, VariableSymbol.Member member) {
+        return layer.getInputTypes();
+    }
+
+    @Override
+    public void checkInput(List<ArchTypeSymbol> inputTypes, LayerSymbol layer, VariableSymbol.Member member) {
+        errorIfInputSizeIsNotOne(inputTypes, layer);
+    }
+
+    public static LayerNorm create(){
+        LayerNorm declaration = new LayerNorm();
+        declaration.setParameters(new ArrayList<>());
+        return declaration;
+    }
+}

src/main/java/de/monticore/lang/monticar/cnnarch/predefined/LoadNetwork.java

@@ -24,7 +24,7 @@ public class LoadNetwork extends PredefinedLayerDeclaration {
 
     @Override
     public List<ArchTypeSymbol> computeOutputTypes(List<ArchTypeSymbol> inputTypes, LayerSymbol layer, VariableSymbol.Member member) {
-        Optional<List<Integer>> optValue = layer.getIntTupleValue(AllPredefinedLayers.OUTPUT_SHAPE_NAME);
+        Optional<List<Integer>> optValue = layer.getIntOrIntTupleValues(AllPredefinedLayers.OUTPUT_SHAPE_NAME);
 
         List<Integer> shapeList = Arrays.asList(1, 1, 1);
 
@@ -61,11 +61,11 @@ public class LoadNetwork extends PredefinedLayerDeclaration {
                         .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.NUM_INPUTS_NAME)
-                        .constraints(Constraints.INTEGER)
+                        .constraints(Constraints.INTEGER, Constraints.POSITIVE)
                         .build(),
                 new ParameterSymbol.Builder()
                         .name(AllPredefinedLayers.OUTPUT_SHAPE_NAME)
-                        .constraints(Constraints.INTEGER_TUPLE)
+                        .constraints(Constraints.INTEGER_OR_INTEGER_TUPLE, Constraints.POSITIVE)
                         .build()));
         declaration.setParameters(parameters);
         return declaration;

src/test/resources/invalid_tests/EpisodicMemoryInvalidPlacement.cnna

@@ -5,11 +5,11 @@ architecture EpisodicMemoryInvalidPlacement{
 
     data ->
     (
-        EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, replayMemoryStoreProb=0.5, localAdaptionGradientSteps=30, maxStoredSamples=-1, localAdaptionK=32, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
+        EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, memoryStoreProb=1, maxStoredSamples=-1, memoryReplacementStrategy="no_replacement", useLocalAdaptation=true, localAdaptationK=32, localAdaptationGradientSteps=30, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
 	    FullyConnected(units=33) ->
         Softmax()
     |
-        EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, replayMemoryStoreProb=0.5, localAdaptionGradientSteps=30, maxStoredSamples=-1, localAdaptionK=32, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
+        EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, memoryStoreProb=1, maxStoredSamples=-1, memoryReplacementStrategy="no_replacement", useLocalAdaptation=true, localAdaptationK=32, localAdaptationGradientSteps=30, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
 	    FullyConnected(units=33) ->
         Softmax()
     ) ->

src/test/resources/valid_tests/EpisodicMemoryNetwork.cnna

@@ -4,11 +4,11 @@ architecture EpisodicMemoryNetwork{
     def output Q(0:1)^{33} softmax
 
     data ->
-    EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, replayMemoryStoreProb=0.5, localAdaptionGradientSteps=30, maxStoredSamples=-1, localAdaptionK=32, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
+    EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, memoryStoreProb=1, maxStoredSamples=-1, memoryReplacementStrategy="no_replacement", useLocalAdaptation=true, localAdaptationK=32, localAdaptationGradientSteps=30, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
     FullyConnected(units=33) ->
-    EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, replayMemoryStoreProb=0.5, localAdaptionGradientSteps=30, maxStoredSamples=-1, localAdaptionK=32, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
+    EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, memoryStoreProb=1, maxStoredSamples=-1, memoryReplacementStrategy="no_replacement", useLocalAdaptation=true, localAdaptationK=32, localAdaptationGradientSteps=30, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
 	FullyConnected(units=33) ->
-    EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, replayMemoryStoreProb=0.5, localAdaptionGradientSteps=30, maxStoredSamples=-1, localAdaptionK=32, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
+    EpisodicMemory(replayInterval=78, replayBatchSize=100, replaySteps=1, replayGradientSteps=1, memoryStoreProb=1, maxStoredSamples=-1, memoryReplacementStrategy="no_replacement", useLocalAdaptation=true, localAdaptationK=32, localAdaptationGradientSteps=30, queryNetDir="pretrained", queryNetPrefix="network_name-", queryNetNumInputs=1) ->
 	FullyConnected(units=33) ->
     Softmax() ->
     softmax;