Commit 35a07603 authored by Evgeny Kusmenko's avatar Evgeny Kusmenko
Browse files

Merge branch 'develop' into 'master'

Implemented layer variables and RNN layer

See merge request !20
parents bc846994 4880932b
Pipeline #171107 passed with stages
in 3 minutes and 51 seconds
......@@ -8,16 +8,16 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-gluon-generator</artifactId>
<version>0.2.6-SNAPSHOT</version>
<version>0.2.7-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<CNNArch.version>0.3.1-SNAPSHOT</CNNArch.version>
<CNNArch.version>0.3.2-SNAPSHOT</CNNArch.version>
<CNNTrain.version>0.3.6-SNAPSHOT</CNNTrain.version>
<CNNArch2X.version>0.0.2-SNAPSHOT</CNNArch2X.version>
<CNNArch2X.version>0.0.3-SNAPSHOT</CNNArch2X.version>
<embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
<EMADL2PythonWrapper.version>0.0.2-SNAPSHOT</EMADL2PythonWrapper.version>
......@@ -238,6 +238,7 @@
<version>2.19.1</version>
<configuration>
<useSystemClassLoader>false</useSystemClassLoader>
<trimStackTrace>false</trimStackTrace>
</configuration>
</plugin>
<plugin>
......
......@@ -20,16 +20,16 @@
*/
package de.monticore.lang.monticar.cnnarch.gluongenerator;
import de.monticore.lang.monticar.cnnarch._symboltable.IOSymbol;
import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.Target;
import de.monticore.lang.monticar.cnnarch.generator.TemplateConfiguration;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.se_rwth.commons.logging.Log;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cmake.CMakeConfig;
import de.monticore.lang.monticar.generator.cmake.CMakeFindModule;
import java.util.HashMap;
import java.util.Map;
import java.util.*;
public class CNNArch2Gluon extends CNNArchGenerator {
......@@ -52,18 +52,9 @@ public class CNNArch2Gluon extends CNNArchGenerator {
} else {
fileContentMap = compileFileContentMap(architecture);
}
checkValidOutputTypes(architecture);
return fileContentMap;
}
private void checkValidOutputTypes(ArchitectureSymbol architecture) {
if (((IOSymbol)architecture.getOutputs().get(0)).getDefinition().getType().getWidth() != 1
|| ((IOSymbol)architecture.getOutputs().get(0)).getDefinition().getType().getHeight() != 1) {
Log.error("This cnn architecture has a multi-dimensional output, which is currently not supported by" +
" the code generator.", architecture.getSourcePosition());
}
}
private Map<String, String> compilePythonFiles(CNNArch2GluonTemplateController controller, ArchitectureSymbol architecture) {
Map<String, String> fileContentMap = new HashMap<>();
Map.Entry<String, String> temp;
......@@ -120,4 +111,20 @@ public class CNNArch2Gluon extends CNNArchGenerator {
architecture, templateConfiguration);
return compilePythonFiles(archTc, architecture);
}
public Map<String, String> generateCMakeContent(String rootModelName) {
// model name should start with a lower case letter. If it is a component, replace dot . by _
rootModelName = rootModelName.replace('.', '_').replace('[', '_').replace(']', '_');
rootModelName = rootModelName.substring(0, 1).toLowerCase() + rootModelName.substring(1);
CMakeConfig cMakeConfig = new CMakeConfig(rootModelName);
cMakeConfig.addModuleDependency(new CMakeFindModule("Armadillo", true));
cMakeConfig.addCMakeCommand("set(LIBS ${LIBS} mxnet)");
Map<String,String> fileContentMap = new HashMap<>();
for (FileContent fileContent : cMakeConfig.generateCMakeFiles()){
fileContentMap.put(fileContent.getFileName(), fileContent.getFileContent());
}
return fileContentMap;
}
}
\ No newline at end of file
......@@ -22,9 +22,19 @@ public class CNNArch2GluonArchitectureSupportChecker extends ArchitectureSupport
return true;
}
@Override
protected boolean checkMultiDimensionalOutput(ArchitectureSymbol architecture) {
return true;
}
@Override
protected boolean checkConstants(ArchitectureSymbol architecture) {
return true;
}
@Override
protected boolean checkLayerVariables(ArchitectureSymbol architecture) {
return true;
}
}
......@@ -23,6 +23,7 @@ public class CNNArch2GluonLayerSupportChecker extends LayerSupportChecker {
supportedLayerList.add(AllPredefinedLayers.CONCATENATE_NAME);
supportedLayerList.add(AllPredefinedLayers.FLATTEN_NAME);
supportedLayerList.add(AllPredefinedLayers.ONE_HOT_NAME);
supportedLayerList.add(AllPredefinedLayers.RNN_NAME);
}
}
......@@ -46,20 +46,25 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
getTemplateConfiguration().processTemplate(ftlContext, templatePath, writer);
}
public void include(IOSymbol ioElement, Writer writer, NetDefinitionMode netDefinitionMode){
public void include(VariableSymbol element, Writer writer, NetDefinitionMode netDefinitionMode){
ArchitectureElementData previousElement = getCurrentElement();
setCurrentElement(ioElement);
setCurrentElement(element);
if (ioElement.isAtomic()){
if (ioElement.isInput()){
include(TEMPLATE_ELEMENTS_DIR_PATH, "Input", writer, netDefinitionMode);
if (element.isAtomic()){
if (element.getType() == VariableSymbol.Type.IO) {
if (element.isInput()){
include(TEMPLATE_ELEMENTS_DIR_PATH, "Input", writer, netDefinitionMode);
}
else {
include(TEMPLATE_ELEMENTS_DIR_PATH, "Output", writer, netDefinitionMode);
}
}
else {
include(TEMPLATE_ELEMENTS_DIR_PATH, "Output", writer, netDefinitionMode);
else if (element.getType() == VariableSymbol.Type.LAYER) {
include(TEMPLATE_ELEMENTS_DIR_PATH, element.getLayerVariableDeclaration().getLayer().getName(), writer, netDefinitionMode);
}
}
else {
include(ioElement.getResolvedThis().get(), writer, netDefinitionMode);
include(element.getResolvedThis().get(), writer, netDefinitionMode);
}
setCurrentElement(previousElement);
......@@ -84,7 +89,6 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
setCurrentElement(layer);
if (layer.isAtomic()){
ArchitectureElementSymbol nextElement = layer.getOutputElement().get();
String templateName = layer.getDeclaration().getName();
include(TEMPLATE_ELEMENTS_DIR_PATH, templateName, writer, netDefinitionMode);
}
......@@ -117,7 +121,7 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
include((ConstantSymbol) architectureElement, writer, netDefinitionMode);
}
else {
include((IOSymbol) architectureElement, writer, netDefinitionMode);
include((VariableSymbol) architectureElement, writer, netDefinitionMode);
}
}
......@@ -132,27 +136,128 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
include(architectureElement, getWriter(), netDefinitionMode);
}
public String ioNameToCpp(String ioName) {
return ioName.replaceAll("_([0-9]+)_", "[$1]");
public Set<String> getStreamInputNames(SerialCompositeElementSymbol stream) {
return getStreamInputs(stream).keySet();
}
public List<String> getStreamInputNames(SerialCompositeElementSymbol stream) {
List<String> names = new ArrayList<>();
public Collection<List<String>> getStreamInputDimensions(SerialCompositeElementSymbol stream) {
return getStreamInputs(stream).values();
}
public Set<String> getStreamOutputNames(SerialCompositeElementSymbol stream) {
Set<String> outputNames = new LinkedHashSet<>();
for (ArchitectureElementSymbol element : stream.getLastAtomicElements()) {
if (element.isOutput()) {
outputNames.add(getName(element));
}
}
outputNames.addAll(getStreamLayerVariableMembers(stream, "1", true).keySet());
return outputNames;
}
// Used to initialize all layer variable members which are passed through the networks
public Map<String, List<String>> getLayerVariableMembers(String batchSize) {
Map<String, List<String>> members = new LinkedHashMap<>();
for (SerialCompositeElementSymbol stream : getArchitecture().getStreams()) {
members.putAll(getStreamLayerVariableMembers(stream, batchSize, true));
}
return members;
}
private Map<String, List<String>> getStreamInputs(SerialCompositeElementSymbol stream) {
Map<String, List<String>> inputs = new LinkedHashMap<>();
for (ArchitectureElementSymbol element : stream.getFirstAtomicElements()) {
names.add(getName(element));
if (element.isInput()) {
List<Integer> intDimensions = element.getOutputTypes().get(0).getDimensions();
List<String> dimensions = new ArrayList<>();
for (Integer intDimension : intDimensions) {
dimensions.add(intDimension.toString());
}
// Add batch size dimension
dimensions.add(0, "1");
inputs.put(getName(element), dimensions);
}
}
return names;
inputs.putAll(getStreamLayerVariableMembers(stream, "1", false));
return inputs;
}
public List<String> getStreamOutputNames(SerialCompositeElementSymbol stream) {
List<String> names = new ArrayList<>();
private Map<String, List<String>> getStreamLayerVariableMembers(SerialCompositeElementSymbol stream, String batchSize, boolean includeOutput) {
Map<String, List<String>> members = new HashMap<>();
for (ArchitectureElementSymbol element : stream.getLastAtomicElements()) {
names.add(getName(element));
List<ArchitectureElementSymbol> elements = stream.getSpannedScope().resolveLocally(ArchitectureElementSymbol.KIND);
for (ArchitectureElementSymbol element : elements) {
if (element instanceof VariableSymbol) {
VariableSymbol variable = (VariableSymbol) element;
if (variable.getType() == VariableSymbol.Type.LAYER && variable.getMember() == VariableSymbol.Member.NONE) {
LayerVariableDeclarationSymbol layerVariableDeclaration = variable.getLayerVariableDeclaration();
if (layerVariableDeclaration.getLayer().getDeclaration().isPredefined()) {
PredefinedLayerDeclaration predefinedLayerDeclaration =
(PredefinedLayerDeclaration) layerVariableDeclaration.getLayer().getDeclaration();
if (predefinedLayerDeclaration.isValidMember(VariableSymbol.Member.STATE)) {
String name = variable.getName() + "_state_";
List<Integer> intDimensions = predefinedLayerDeclaration.computeOutputTypes(
layerVariableDeclaration.getLayer().getInputTypes(),
layerVariableDeclaration.getLayer(),
VariableSymbol.Member.STATE
).get(0).getDimensions();
List<String> dimensions = new ArrayList<>();
for (Integer intDimension : intDimensions) {
dimensions.add(intDimension.toString());
}
// Add batch size dimension at index 1, since RNN states in Gluon have the format
// (layers, batch_size, units)
dimensions.add(1, batchSize);
members.put(name, dimensions);
}
if (includeOutput) {
if (predefinedLayerDeclaration.isValidMember(VariableSymbol.Member.OUTPUT)) {
String name = variable.getName() + "_output_";
List<Integer> intDimensions = predefinedLayerDeclaration.computeOutputTypes(
layerVariableDeclaration.getLayer().getInputTypes(),
layerVariableDeclaration.getLayer(),
VariableSymbol.Member.OUTPUT
).get(0).getDimensions();
List<String> dimensions = new ArrayList<>();
for (Integer intDimension : intDimensions) {
dimensions.add(intDimension.toString());
}
// Add batch size dimension at index 0, since we use NTC format for RNN output in Gluon
dimensions.add(0, batchSize);
members.put(name, dimensions);
}
}
}
}
}
}
return names;
return members;
}
}
package de.monticore.lang.monticar.cnnarch.gluongenerator.annotations;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.IOSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.IODeclarationSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.VariableSymbol;
import de.monticore.lang.monticar.cnntrain._symboltable.NNArchitectureSymbol;
import de.monticore.lang.monticar.cnntrain.annotations.Range;
import de.monticore.lang.monticar.ranges._ast.ASTRange;
......@@ -42,19 +43,19 @@ public class ArchitectureAdapter extends NNArchitectureSymbol {
@Override
public Map<String, List<Integer>> getDimensions() {
return getAllIOSymbols().stream().collect(Collectors.toMap(CommonSymbol::getName,
s-> s.getDefinition().getType().getDimensions()));
s-> ((IODeclarationSymbol) s.getDeclaration()).getType().getDimensions()));
}
@Override
public Map<String, Range> getRanges() {
return getAllIOSymbols().stream().collect(Collectors.toMap(CommonSymbol::getName,
s -> astRangeToTrainRange(s.getDefinition().getType().getDomain().getRangeOpt().orElse(null))));
s -> astRangeToTrainRange(((IODeclarationSymbol) s.getDeclaration()).getType().getDomain().getRangeOpt().orElse(null))));
}
@Override
public Map<String, String> getTypes() {
return getAllIOSymbols().stream().collect(Collectors.toMap(CommonSymbol::getName,
s -> s.getDefinition().getType().getDomain().getName()));
s -> ((IODeclarationSymbol) s.getDeclaration()).getType().getDomain().getName()));
}
public ArchitectureSymbol getArchitectureSymbol() {
......@@ -77,18 +78,18 @@ public class ArchitectureAdapter extends NNArchitectureSymbol {
}
}
private List<IOSymbol> getIOOutputSymbols() {
private List<VariableSymbol> getIOOutputSymbols() {
return architectureSymbol.getOutputs();
}
private List<IOSymbol> getIOInputSymbols() {
private List<VariableSymbol> getIOInputSymbols() {
return architectureSymbol.getInputs();
}
private List<IOSymbol> getAllIOSymbols() {
List<IOSymbol> ioSymbols = new ArrayList<>();
private List<VariableSymbol> getAllIOSymbols() {
List<VariableSymbol> ioSymbols = new ArrayList<>();
ioSymbols.addAll(getIOOutputSymbols());
ioSymbols.addAll(getIOInputSymbols());
return ioSymbols;
}
}
}
\ No newline at end of file
......@@ -2,7 +2,7 @@ import mxnet as mx
import logging
import os
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
<#if stream.isTrainable()>
from CNNNet_${tc.fullArchitectureName} import Net_${stream?index}
</#if>
</#list>
......@@ -52,11 +52,11 @@ class ${tc.fileNameWithoutEnding}:
def construct(self, context, data_mean=None, data_std=None):
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
<#if stream.isTrainable()>
self.networks[${stream?index}] = Net_${stream?index}(data_mean=data_mean, data_std=data_std)
self.networks[${stream?index}].collect_params().initialize(self.weight_initializer, ctx=context)
self.networks[${stream?index}].hybridize()
self.networks[${stream?index}](<#list stream.getFirstAtomicElements() as input>mx.nd.zeros((1, ${tc.join(input.definition.type.dimensions, ",")},), ctx=context)<#sep>, </#list>)
self.networks[${stream?index}](<#list tc.getStreamInputDimensions(stream) as dimensions>mx.nd.zeros((${tc.join(dimensions, ",")},), ctx=context)<#sep>, </#list>)
</#if>
</#list>
......
......@@ -6,8 +6,8 @@ import sys
from mxnet import nd
class ${tc.fileNameWithoutEnding}:
_input_names_ = [${tc.join(tc.architectureInputs, ",", "'", "'")}]
_output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "_label'")}]
_input_names_ = [<#list tc.architectureInputs as inputName>'${inputName?keep_before_last("_")}'<#sep>, </#list>]
_output_names_ = [${tc.join(tc.architectureOutputs, ",", "'", "label'")}]
def __init__(self):
self._data_dir = "${tc.dataPath}/"
......@@ -21,8 +21,8 @@ class ${tc.fileNameWithoutEnding}:
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
train_label = {}
for output_name in self._output_names_:
......
......@@ -79,7 +79,7 @@ class NoNormalization(gluon.HybridBlock):
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
<#if stream.isTrainable()>
class Net_${stream?index}(gluon.HybridBlock):
def __init__(self, data_mean=None, data_std=None, **kwargs):
super(Net_${stream?index}, self).__init__(**kwargs)
......@@ -88,12 +88,8 @@ class Net_${stream?index}(gluon.HybridBlock):
${tc.include(stream, "ARCHITECTURE_DEFINITION")}
def hybrid_forward(self, F, ${tc.join(tc.getStreamInputNames(stream), ", ")}):
outputs = []
${tc.include(stream, "FORWARD_FUNCTION")}
<#if tc.getStreamOutputNames(stream)?size gt 1>
return tuple(outputs)
<#else>
return outputs[0]
</#if>
return ${tc.join(tc.getStreamOutputNames(stream), ", ")}
</#if>
</#list>
\ No newline at end of file
......@@ -10,19 +10,19 @@
#include <CNNBufferFile.h>
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
<#if stream.isTrainable()>
class ${tc.fileNameWithoutEnding}_${stream?index}{
public:
const std::string json_file = "model/${tc.componentName}/model_${stream?index}_newest-symbol.json";
const std::string param_file = "model/${tc.componentName}/model_${stream?index}_newest-0000.params";
const std::vector<std::string> input_keys = {
<#if (tc.getStreamInputNames(stream)?size == 1)>
<#if tc.getStreamInputNames(stream)?size == 1>
"data"
<#else>
<#list tc.getStreamInputNames(stream) as inputName>"data${inputName?index}"<#sep>, </#list>
<#list tc.getStreamInputNames(stream) as variable>"data${variable?index}"<#sep>, </#list>
</#if>
};
const std::vector<std::vector<mx_uint>> input_shapes = {<#list stream.getFirstAtomicElements() as input>{1, ${tc.join(input.definition.type.dimensions, ", ")}}<#sep>, </#list>};
const std::vector<std::vector<mx_uint>> input_shapes = {<#list tc.getStreamInputDimensions(stream) as dimensions>{${tc.join(dimensions, ", ")}}<#sep>, </#list>};
const bool use_gpu = false;
PredictorHandle handle;
......@@ -35,14 +35,10 @@ public:
if(handle) MXPredFree(handle);
}
void predict(${tc.join(tc.getStreamInputNames(stream), ", ", "const std::vector<float> &", "")},
${tc.join(tc.getStreamOutputNames(stream), ", ", "std::vector<float> &", "")}){
<#list tc.getStreamInputNames(stream) as inputName>
<#if (tc.getStreamInputNames(stream)?size == 1)>
MXPredSetInput(handle, "data", ${inputName}.data(), static_cast<mx_uint>(${inputName}.size()));
<#else>
MXPredSetInput(handle, "data${inputName?index}", ${inputName}.data(), static_cast<mx_uint>(${inputName}.size()));
</#if>
void predict(${tc.join(tc.getStreamInputNames(stream), ", ", "const std::vector<float> &in_", "")},
${tc.join(tc.getStreamOutputNames(stream), ", ", "std::vector<float> &out_", "")}){
<#list tc.getStreamInputNames(stream) as variable>
MXPredSetInput(handle, input_keys[${variable?index}].c_str(), in_${variable}.data(), static_cast<mx_uint>(in_${variable}.size()));
</#list>
MXPredForward(handle);
......@@ -52,13 +48,13 @@ public:
mx_uint shape_len;
size_t size;
<#list tc.getStreamOutputNames(stream) as outputName>
output_index = ${outputName?index?c};
<#list tc.getStreamOutputNames(stream) as variable>
output_index = ${variable?index?c};
MXPredGetOutputShape(handle, output_index, &shape, &shape_len);
size = 1;
for (mx_uint i = 0; i < shape_len; ++i) size *= shape[i];
assert(size == ${outputName}.size());
MXPredGetOutput(handle, ${outputName?index?c}, &(${outputName}[0]), ${outputName}.size());
assert(size == out_${variable}.size());
MXPredGetOutput(handle, ${variable?index?c}, &(out_${variable}[0]), out_${variable}.size());
</#list>
}
......
......@@ -133,28 +133,18 @@ class ${tc.fileNameWithoutEnding}:
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
<#list tc.architectureInputs as input_name>
${input_name}_data = batch.data[${input_name?index}].as_in_context(mx_context)
${input_name} = batch.data[${input_name?index}].as_in_context(mx_context)
</#list>
<#list tc.architectureOutputs as output_name>
${output_name}_label = batch.label[${output_name?index}].as_in_context(mx_context)
${output_name}label = batch.label[${output_name?index}].as_in_context(mx_context)
</#list>
with autograd.record():
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
${tc.join(tc.getStreamOutputNames(stream), ", ", "", "_output")} = self._networks[${stream?index}](${tc.join(tc.getStreamInputNames(stream), ", ", "", "_data")})
<#else>
${tc.include(stream, "PYTHON_INLINE")}
</#if>
</#list>
<#include "pythonExecute.ftl">
loss = \
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
<#list tc.getStreamOutputNames(stream) as output_name>
loss_function(${output_name}_output, ${output_name}_label)<#sep> + \
</#list><#sep> + \
</#if>
<#list tc.architectureOutputs as output_name>
loss_function(${output_name}, ${output_name}label)<#sep> + \
</#list>
......@@ -182,7 +172,7 @@ ${tc.include(stream, "PYTHON_INLINE")}
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(train_iter):
<#list tc.architectureInputs as input_name>
${input_name}_data = batch.data[${input_name?index}].as_in_context(mx_context)
${input_name} = batch.data[${input_name?index}].as_in_context(mx_context)
</#list>
labels = [
......@@ -192,18 +182,12 @@ ${tc.include(stream, "PYTHON_INLINE")}
]
if True: # Fix indentation
<#list tc.architecture.streams as stream>
<#if stream.isNetwork()>
${tc.join(tc.getStreamOutputNames(stream), ", ", "", "_output")} = self._networks[${stream?index}](${tc.join(tc.getStreamInputNames(stream), ", ", "", "_data")})
<#else>
${tc.include(stream, "PYTHON_INLINE")}
</#if>
</#list>
if True: <#-- Fix indentation -->
<#include "pythonExecute.ftl">
predictions = [
<#list tc.architectureOutputs as output_name>
mx.nd.argmax(${output_name}_output, axis=1)<#sep>,
mx.nd.argmax(${output_name}, axis=1)<#sep>,
</#list>
]
......@@ -215,7 +199,7 @@ ${tc.include(stream, "PYTHON_INLINE")}
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(test_iter):
<#list tc.architectureInputs as input_name>
${input_name}_data = batch.data[${input_name?index}].as_in_context(mx_context)
${input_name} = batch.data[${input_name?index}].as_in_context(mx_context)
</#list>
labels = [
......@@ -225,18 +209,12 @@ ${tc.include(stream, "PYTHON_INLINE")}
]
if True: # Fix indentation