Commit 18278bb6 authored by Sebastian N.'s avatar Sebastian N.
Browse files

Merge rnn into develop

parents 053bf612 74d278fe
Pipeline #150615 canceled with stages
......@@ -15,9 +15,9 @@
<properties>
<!-- .. SE-Libraries .................................................. -->
<CNNArch.version>0.3.0-SNAPSHOT</CNNArch.version>
<CNNArch.version>0.3.1-SNAPSHOT</CNNArch.version>
<CNNTrain.version>0.3.2-SNAPSHOT</CNNTrain.version>
<CNNArch2MXNet.version>0.2.15-SNAPSHOT</CNNArch2MXNet.version>
<CNNArch2MXNet.version>0.2.16-SNAPSHOT</CNNArch2MXNet.version>
<embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
<EMADL2PythonWrapper.version>0.0.1</EMADL2PythonWrapper.version>
......@@ -116,6 +116,12 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-rules</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
......
......@@ -33,11 +33,15 @@ import java.util.Map;
public class CNNArch2Gluon extends CNNArch2MxNet {
public CNNArch2Gluon() {
architectureSupportChecker = new CNNArch2GluonArchitectureSupportChecker();
layerSupportChecker = new CNNArch2GluonLayerSupportChecker();
}
//check cocos with CNNArchCocos.checkAll(architecture) before calling this method.
@Override
public Map<String, String> generateStrings(ArchitectureSymbol architecture){
Map<String, String> fileContentMap = compileFileContentMap(architecture);
checkValidGeneration(architecture);
return fileContentMap;
}
......@@ -85,6 +89,9 @@ public class CNNArch2Gluon extends CNNArch2MxNet {
temp = controller.process("CNNPredictor", Target.CPP);
fileContentMap.put(temp.getKey(), temp.getValue());
temp = controller.process("CNNSupervisedTrainer", Target.PYTHON);
fileContentMap.put(temp.getKey(), temp.getValue());
temp = controller.process("execute", Target.CPP);
fileContentMap.put(temp.getKey().replace(".h", ""), temp.getValue());
......
package de.monticore.lang.monticar.cnnarch.gluongenerator;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.ArchitectureSupportChecker;
public class CNNArch2GluonArchitectureSupportChecker extends ArchitectureSupportChecker {
public CNNArch2GluonArchitectureSupportChecker() {}
/*protected boolean checkMultipleStreams(ArchitectureSymbol architecture) {
return true;
}*/
protected boolean checkMultipleInputs(ArchitectureSymbol architecture) {
return true;
}
protected boolean checkMultipleOutputs(ArchitectureSymbol architecture) {
return true;
}
}
package de.monticore.lang.monticar.cnnarch.gluongenerator;
import de.monticore.lang.monticar.cnnarch.predefined.AllPredefinedLayers;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.LayerSupportChecker;
public class CNNArch2GluonLayerSupportChecker extends LayerSupportChecker {
public CNNArch2GluonLayerSupportChecker() {
supportedLayerList.add(AllPredefinedLayers.FULLY_CONNECTED_NAME);
supportedLayerList.add(AllPredefinedLayers.CONVOLUTION_NAME);
supportedLayerList.add(AllPredefinedLayers.SOFTMAX_NAME);
supportedLayerList.add(AllPredefinedLayers.SIGMOID_NAME);
supportedLayerList.add(AllPredefinedLayers.TANH_NAME);
supportedLayerList.add(AllPredefinedLayers.RELU_NAME);
supportedLayerList.add(AllPredefinedLayers.DROPOUT_NAME);
supportedLayerList.add(AllPredefinedLayers.POOLING_NAME);
supportedLayerList.add(AllPredefinedLayers.GLOBAL_POOLING_NAME);
supportedLayerList.add(AllPredefinedLayers.LRN_NAME);
supportedLayerList.add(AllPredefinedLayers.BATCHNORM_NAME);
supportedLayerList.add(AllPredefinedLayers.SPLIT_NAME);
supportedLayerList.add(AllPredefinedLayers.GET_NAME);
supportedLayerList.add(AllPredefinedLayers.ADD_NAME);
supportedLayerList.add(AllPredefinedLayers.CONCATENATE_NAME);
supportedLayerList.add(AllPredefinedLayers.FLATTEN_NAME);
supportedLayerList.add(AllPredefinedLayers.ONE_HOT_NAME);
}
}
......@@ -71,7 +71,7 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
if (layer.isAtomic()){
ArchitectureElementSymbol nextElement = layer.getOutputElement().get();
if (!isSoftmaxOutput(nextElement) && !isLogisticRegressionOutput(nextElement)){
if (!isSoftmaxOutput(nextElement) && !isLogisticRegressionOutput(nextElement) && !isOneHotOutput(nextElement)){
String templateName = layer.getDeclaration().getName();
include(TEMPLATE_ELEMENTS_DIR_PATH, templateName, writer, netDefinitionMode);
}
......
......@@ -114,9 +114,6 @@ public class CNNTrain2Gluon extends CNNTrain2MxNet {
if (configData.isSupervisedLearning()) {
String cnnTrainTemplateContent = templateConfiguration.processTemplate(ftlContext, "CNNTrainer.ftl");
fileContentMap.put("CNNTrainer_" + getInstanceName() + ".py", cnnTrainTemplateContent);
String cnnSupervisedTrainerContent = templateConfiguration.processTemplate(ftlContext, "CNNSupervisedTrainer.ftl");
fileContentMap.put("supervised_trainer.py", cnnSupervisedTrainerContent);
} else if (configData.isReinforcementLearning()) {
final String trainerName = "CNNTrainer_" + getInstanceName();
final RLAlgorithm rlAlgorithm = configData.getRlAlgorithm();
......
......@@ -3,9 +3,10 @@ package de.monticore.lang.monticar.cnnarch.gluongenerator.reinforcement.critic;
import com.google.common.collect.Lists;
import de.monticore.lang.monticar.cnnarch._symboltable.*;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2Gluon;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2GluonArchitectureSupportChecker;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2GluonLayerSupportChecker;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArchSymbolCompiler;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.TemplateConfiguration;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.checker.AllowAllLayerSupportChecker;
import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
import de.monticore.lang.monticar.cnntrain.annotations.Range;
import de.monticore.lang.monticar.cnntrain.annotations.TrainedArchitecture;
......@@ -81,7 +82,8 @@ public class CriticNetworkGenerator {
gluonGenerator.setGenerationTargetPath(this.getGenerationTargetPath());
Map<String, String> fileContentMap = new HashMap<>();
CNNArchSymbolCompiler symbolCompiler = new CNNArchSymbolCompiler(new AllowAllLayerSupportChecker());
CNNArchSymbolCompiler symbolCompiler = new CNNArchSymbolCompiler(new CNNArch2GluonArchitectureSupportChecker(),
new CNNArch2GluonLayerSupportChecker());
ArchitectureSymbol architectureSymbol = symbolCompiler.compileArchitectureSymbolFromModelsDir(directoryOfCnnArchFile, criticNetworkName);
architectureSymbol.setComponentName(criticNetworkName);
fileContentMap.putAll(gluonGenerator.generateStringsAllowMultipleIO(architectureSymbol, true));
......
......@@ -6,7 +6,7 @@ from CNNNet_${tc.fullArchitectureName} import Net
class ${tc.fileNameWithoutEnding}:
_model_dir_ = "model/${tc.componentName}/"
_model_prefix_ = "model"
_input_shapes_ = [<#list tc.architecture.inputs as input>(${tc.join(input.definition.type.dimensions, ",")},)<#if input?has_next>,</#if></#list>]
_input_shapes_ = [<#list tc.architecture.inputs as input>(${tc.join(input.definition.type.dimensions, ",")},)<#sep>, </#list>]
def __init__(self):
self.weight_initializer = mx.init.Normal()
......@@ -43,12 +43,11 @@ class ${tc.fileNameWithoutEnding}:
self.net.load_parameters(self._model_dir_ + param_file)
return lastEpoch
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
self.net.hybridize()
self.net(<#list tc.architecture.inputs as input>mx.nd.zeros((1,)+self._input_shapes_[${input?index}], ctx=context)<#if input?has_next>,</#if></#list>)
self.net(<#list tc.architecture.inputs as input>mx.nd.zeros((1,) + self._input_shapes_[${input?index}], ctx=context)<#sep>, </#list>)
if not os.path.exists(self._model_dir_):
os.makedirs(self._model_dir_)
......
......@@ -3,8 +3,9 @@ import h5py
import mxnet as mx
import logging
import sys
from mxnet import nd
class ${tc.fullArchitectureName}DataLoader:
class ${tc.fileNameWithoutEnding}:
_input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
_output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
......@@ -14,21 +15,38 @@ class ${tc.fullArchitectureName}DataLoader:
def load_data(self, batch_size):
train_h5, test_h5 = self.load_h5_files()
data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
train_data = {}
data_mean = {}
data_std = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
train_label = {}
for output_name in self._output_names_:
train_label[output_name] = train_h5[output_name]
train_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=batch_size)
train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
train_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
test_iter = None
if test_h5 != None:
test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
test_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
test_data = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
test_label = {}
for output_name in self._output_names_:
test_label[output_name] = test_h5[output_name]
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=batch_size)
return train_iter, test_iter, data_mean, data_std
def load_h5_files(self):
......@@ -36,21 +54,39 @@ class ${tc.fullArchitectureName}DataLoader:
test_h5 = None
train_path = self._data_dir + "train.h5"
test_path = self._data_dir + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
test_iter = None
for input_name in self._input_names_:
if not input_name in train_h5:
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the dataset "
+ "'" + input_name + "'")
sys.exit(1)
for output_name in self._output_names_:
if not output_name in train_h5:
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the dataset "
+ "'" + output_name + "'")
sys.exit(1)
if os.path.isfile(test_path):
test_h5 = h5py.File(test_path, 'r')
if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
for input_name in self._input_names_:
if not input_name in test_h5:
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the dataset "
+ "'" + input_name + "'")
sys.exit(1)
for output_name in self._output_names_:
if not output_name in test_h5:
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the dataset "
+ "'" + output_name + "'")
sys.exit(1)
else:
logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
return train_h5, test_h5
else:
logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
......
......@@ -2,6 +2,16 @@ import mxnet as mx
import numpy as np
from mxnet import gluon
class OneHot(gluon.HybridBlock):
def __init__(self, size, **kwargs):
super(OneHot, self).__init__(**kwargs)
with self.name_scope():
self.size = size
def hybrid_forward(self, F, x):
return F.one_hot(indices=F.argmax(data=x, axis=1), depth=self.size)
class Softmax(gluon.HybridBlock):
def __init__(self, **kwargs):
super(Softmax, self).__init__(**kwargs)
......@@ -71,8 +81,15 @@ class NoNormalization(gluon.HybridBlock):
class Net(gluon.HybridBlock):
def __init__(self, data_mean=None, data_std=None, **kwargs):
super(Net, self).__init__(**kwargs)
self.last_layers = {}
with self.name_scope():
${tc.include(tc.architecture.body, "ARCHITECTURE_DEFINITION")}
def hybrid_forward(self, F, <#list tc.architecture.inputs as input>${input}<#if input?has_next>, </#if></#list>):
${tc.include(tc.architecture.body, "FORWARD_FUNCTION")}
\ No newline at end of file
${tc.include(tc.architecture.streams[0], "ARCHITECTURE_DEFINITION")}
def hybrid_forward(self, F, ${tc.join(tc.architectureInputs, ", ")}):
<#if tc.architectureOutputs?size gt 1>
outputs = []
</#if>
${tc.include(tc.architecture.streams[0], "FORWARD_FUNCTION")}
<#if tc.architectureOutputs?size gt 1>
return tuple(outputs)
</#if>
......@@ -13,9 +13,14 @@ class ${tc.fileNameWithoutEnding}{
public:
const std::string json_file = "model/${tc.componentName}/model_newest-symbol.json";
const std::string param_file = "model/${tc.componentName}/model_newest-0000.params";
//const std::vector<std::string> input_keys = {"data"};
const std::vector<std::string> input_keys = {${tc.join(tc.architectureInputs, ",", "\"", "\"")}};
const std::vector<std::vector<mx_uint>> input_shapes = {<#list tc.architecture.inputs as input>{1,${tc.join(input.definition.type.dimensions, ",")}}<#if input?has_next>,</#if></#list>};
const std::vector<std::string> input_keys = {
<#if (tc.architectureInputs?size == 1)>
"data"
<#else>
<#list tc.architectureInputs as inputName>"data${inputName?index}"<#sep>, </#list>
</#if>
};
const std::vector<std::vector<mx_uint>> input_shapes = {<#list tc.architecture.inputs as input>{1, ${tc.join(input.definition.type.dimensions, ", ")}}<#sep>, </#list>};
const bool use_gpu = false;
PredictorHandle handle;
......@@ -31,7 +36,11 @@ public:
void predict(${tc.join(tc.architectureInputs, ", ", "const std::vector<float> &", "")},
${tc.join(tc.architectureOutputs, ", ", "std::vector<float> &", "")}){
<#list tc.architectureInputs as inputName>
<#if (tc.architectureInputs?size == 1)>
MXPredSetInput(handle, "data", ${inputName}.data(), static_cast<mx_uint>(${inputName}.size()));
<#else>
MXPredSetInput(handle, "data${inputName?index}", ${inputName}.data(), static_cast<mx_uint>(${inputName}.size()));
</#if>
</#list>
MXPredForward(handle);
......@@ -71,22 +80,17 @@ public:
const mx_uint num_input_nodes = input_keys.size();
<#if (tc.architectureInputs?size >= 2)>
const char* input_keys_ptr[num_input_nodes];
for(mx_uint i = 0; i < num_input_nodes; i++){
input_keys_ptr[i] = input_keys[i].c_str();
}
<#else>
const char* input_key[1] = { "data" };
const char** input_keys_ptr = input_key;
</#if>
mx_uint shape_data_size = 0;
mx_uint input_shape_indptr[input_shapes.size() + 1];
input_shape_indptr[0] = 0;
for(mx_uint i = 0; i < input_shapes.size(); i++){
input_shape_indptr[i+1] = input_shapes[i].size();
shape_data_size += input_shapes[i].size();
input_shape_indptr[i+1] = shape_data_size;
}
mx_uint input_shape_data[shape_data_size];
......
......@@ -6,7 +6,7 @@ import os
import shutil
from mxnet import gluon, autograd, nd
class CNNSupervisedTrainer(object):
class ${tc.fileNameWithoutEnding}:
def __init__(self, data_loader, net_constructor, net=None):
self._data_loader = data_loader
self._net_creator = net_constructor
......@@ -48,7 +48,7 @@ class CNNSupervisedTrainer(object):
if self._net is None:
if normalize:
self._net_creator.construct(
context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
context=mx_context, data_mean=data_mean, data_std=data_std)
else:
self._net_creator.construct(context=mx_context)
......@@ -69,15 +69,18 @@ class CNNSupervisedTrainer(object):
trainer = mx.gluon.Trainer(self._net.collect_params(), optimizer, optimizer_params)
if self._net.last_layer == 'softmax':
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
elif self._net.last_layer == 'sigmoid':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif self._net.last_layer == 'linear':
loss_function = mx.gluon.loss.L2Loss()
else: # TODO: Change default?
loss_function = mx.gluon.loss.L2Loss()
logging.warning("Invalid last_layer, defaulting to L2 loss")
loss_functions = {}
for output_name, last_layer in self._net.last_layers.items():
if last_layer == 'softmax':
loss_functions[output_name] = mx.gluon.loss.SoftmaxCrossEntropyLoss()
elif last_layer == 'sigmoid':
loss_functions[output_name] = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif last_layer == 'linear':
loss_functions[output_name] = mx.gluon.loss.L2Loss()
else:
loss_functions[output_name] = mx.gluon.loss.L2Loss()
logging.warning("Invalid last layer, defaulting to L2 loss")
speed_period = 50
tic = None
......@@ -85,11 +88,17 @@ class CNNSupervisedTrainer(object):
for epoch in range(begin_epoch, begin_epoch + num_epoch):
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
<#list tc.architectureInputs as input_name>
${input_name}_data = batch.data[${input_name?index}].as_in_context(mx_context)
</#list>
<#list tc.architectureOutputs as output_name>
${output_name}_label = batch.label[${output_name?index}].as_in_context(mx_context)
</#list>
with autograd.record():
output = self._net(data)
loss = loss_function(output, label)
${tc.join(tc.architectureOutputs, ", ", "", "_output")} = self._net(${tc.join(tc.architectureInputs, ", ", "", "_data")})
loss = <#list tc.architectureOutputs as output_name>loss_functions['${output_name}'](${output_name}_output, ${output_name}_label)<#sep> + </#list>
loss.backward()
trainer.step(batch_size)
......@@ -112,21 +121,41 @@ class CNNSupervisedTrainer(object):
train_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self._net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
<#list tc.architectureInputs as input_name>
${input_name}_data = batch.data[${input_name?index}].as_in_context(mx_context)
</#list>
labels = [
<#list tc.architectureOutputs as output_name>batch.label[${output_name?index}].as_in_context(mx_context)<#sep>, </#list>
]
${tc.join(tc.architectureOutputs, ", ", "", "_output")} = self._net(${tc.join(tc.architectureInputs, ", ", "", "_data")})
predictions = [
<#list tc.architectureOutputs as output_name>mx.nd.argmax(${output_name}_output, axis=1)<#sep>, </#list>
]
metric.update(preds=predictions, labels=labels)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(test_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self._net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
<#list tc.architectureInputs as input_name>
${input_name}_data = batch.data[${input_name?index}].as_in_context(mx_context)
</#list>
labels = [
<#list tc.architectureOutputs as output_name>batch.label[${output_name?index}].as_in_context(mx_context)<#sep>, </#list>
]
${tc.join(tc.architectureOutputs, ", ", "", "_output")} = self._net(${tc.join(tc.architectureInputs, ", ", "", "_data")})
predictions = [
<#list tc.architectureOutputs as output_name>mx.nd.argmax(${output_name}_output, axis=1)<#sep>, </#list>
]
metric.update(preds=predictions, labels=labels)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
......
import logging
import mxnet as mx
import supervised_trainer
<#list configurations as config>
import CNNCreator_${config.instanceName}
import CNNDataLoader_${config.instanceName}
import CNNSupervisedTrainer_${config.instanceName}
</#list>
if __name__ == "__main__":
......@@ -14,9 +14,11 @@ if __name__ == "__main__":
<#list configurations as config>
${config.instanceName}_creator = CNNCreator_${config.instanceName}.CNNCreator_${config.instanceName}()
${config.instanceName}_loader = CNNDataLoader_${config.instanceName}.${config.instanceName}DataLoader()
${config.instanceName}_trainer = supervised_trainer.CNNSupervisedTrainer(${config.instanceName}_loader,
${config.instanceName}_creator)
${config.instanceName}_loader = CNNDataLoader_${config.instanceName}.CNNDataLoader_${config.instanceName}()
${config.instanceName}_trainer = CNNSupervisedTrainer_${config.instanceName}.CNNSupervisedTrainer_${config.instanceName}(
${config.instanceName}_loader,
${config.instanceName}_creator
)
${config.instanceName}_trainer.train(
<#if (config.batchSize)??>
......
<#assign mode = definition_mode.toString()>
<#if mode == "ARCHITECTURE_DEFINITION">
if not data_mean is None:
assert(not data_std is None)
self.${element.name}_input_normalization = ZScoreNormalization(data_mean=data_mean, data_std=data_std)
if data_mean:
assert(data_std)
self.input_normalization_${element.name} = ZScoreNormalization(data_mean=data_mean['${element.name}'],
data_std=data_std['${element.name}'])
else:
self.${element.name}_input_normalization = NoNormalization()
self.input_normalization_${element.name} = NoNormalization()
</#if>
<#if mode == "FORWARD_FUNCTION">
${element.name} = self.${element.name}_input_normalization(${element.name})
${element.name} = self.input_normalization_${element.name}(${element.name})
</#if>
\ No newline at end of file
<#assign input = element.inputs[0]>
<#assign mode = definition_mode.toString()>
<#assign size = element.size>
<#if mode == "ARCHITECTURE_DEFINITION">
self.${element.name} = OneHot(size=${size})
<#include "OutputShape.ftl">
</#if>
<#if mode == "FORWARD_FUNCTION">
${element.name} = self.${element.name}(${input})
</#if>
<#assign input = element.inputs[0]>
<#assign mode = definition_mode.toString()>
<#if mode == "ARCHITECTURE_DEFINITION">
<#if element.softmaxOutput>
self.last_layer = 'softmax'
self.last_layers['${element.name}'] = 'softmax'
<#elseif element.logisticRegressionOutput>
self.last_layer = 'sigmoid'
self.last_layers['${element.name}'] = 'sigmoid'
<#elseif element.linearRegressionOutput>
self.last_layer = 'linear'
self.last_layers['${element.name}'] = 'linear'
<#elseif element.oneHotOutput>
self.last_layers['${element.name}'] = 'softmax'
</#if>
</#if>
<#if mode == "FORWARD_FUNCTION">
<#if tc.architectureOutputs?size gt 1>
outputs.append(${input})
<#else>
return ${input}