new version

parent 830ed23b
......@@ -30,7 +30,7 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnn-arch</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.1-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnnarch.generator;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchTypeSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureElementSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.LayerSymbol;
import de.monticore.lang.monticar.cnnarch.predefined.AllPredefinedLayers;
import de.se_rwth.commons.logging.Log;
import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.List;
public class ArchitectureElementData {
private String name;
private ArchitectureElementSymbol element;
private CNNArchTemplateController templateController;
public ArchitectureElementData(String name, ArchitectureElementSymbol element, CNNArchTemplateController templateController) {
this.name = name;
this.element = element;
this.templateController = templateController;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public ArchitectureElementSymbol getElement() {
return element;
}
public void setElement(ArchitectureElementSymbol element) {
this.element = element;
}
public CNNArchTemplateController getTemplateController() {
return templateController;
}
public void setTemplateController(CNNArchTemplateController templateController) {
this.templateController = templateController;
}
public List<String> getInputs(){
return getTemplateController().getLayerInputs(getElement());
}
public boolean isLogisticRegressionOutput(){
return getTemplateController().isLogisticRegressionOutput(getElement());
}
public boolean isLinearRegressionOutput(){
boolean result = getTemplateController().isLinearRegressionOutput(getElement());
if (result){
Log.warn("The Output '" + getElement().getName() + "' is a linear regression output (squared loss) during training" +
" because the previous architecture element is not a softmax (cross-entropy loss) or sigmoid (logistic regression loss) activation. " +
"Other loss functions are currently not supported. "
, getElement().getSourcePosition());
}
return result;
}
public boolean isSoftmaxOutput(){
return getTemplateController().isSoftmaxOutput(getElement());
}
public List<Integer> getKernel(){
return ((LayerSymbol) getElement())
.getIntTupleValue(AllPredefinedLayers.KERNEL_NAME).get();
}
public int getChannels(){
return ((LayerSymbol) getElement())
.getIntValue(AllPredefinedLayers.CHANNELS_NAME).get();
}
public List<Integer> getStride(){
return ((LayerSymbol) getElement())
.getIntTupleValue(AllPredefinedLayers.STRIDE_NAME).get();
}
public int getUnits(){
return ((LayerSymbol) getElement())
.getIntValue(AllPredefinedLayers.UNITS_NAME).get();
}
public boolean getNoBias(){
return ((LayerSymbol) getElement())
.getBooleanValue(AllPredefinedLayers.NOBIAS_NAME).get();
}
public double getP(){
return ((LayerSymbol) getElement())
.getDoubleValue(AllPredefinedLayers.P_NAME).get();
}
public int getIndex(){
return ((LayerSymbol) getElement())
.getIntValue(AllPredefinedLayers.INDEX_NAME).get();
}
public int getNumOutputs(){
return ((LayerSymbol) getElement())
.getIntValue(AllPredefinedLayers.NUM_SPLITS_NAME).get();
}
public boolean getFixGamma(){
return ((LayerSymbol) getElement())
.getBooleanValue(AllPredefinedLayers.FIX_GAMMA_NAME).get();
}
public int getNsize(){
return ((LayerSymbol) getElement())
.getIntValue(AllPredefinedLayers.NSIZE_NAME).get();
}
public double getKnorm(){
return ((LayerSymbol) getElement())
.getDoubleValue(AllPredefinedLayers.KNORM_NAME).get();
}
public double getAlpha(){
return ((LayerSymbol) getElement())
.getDoubleValue(AllPredefinedLayers.ALPHA_NAME).get();
}
public double getBeta(){
return ((LayerSymbol) getElement())
.getDoubleValue(AllPredefinedLayers.BETA_NAME).get();
}
@Nullable
public String getPoolType(){
return ((LayerSymbol) getElement())
.getStringValue(AllPredefinedLayers.POOL_TYPE_NAME).get();
}
@Nullable
public List<Integer> getPadding(){
return getPadding((LayerSymbol) getElement());
}
@Nullable
public List<Integer> getPadding(LayerSymbol layer){
List<Integer> kernel = layer.getIntTupleValue(AllPredefinedLayers.KERNEL_NAME).get();
List<Integer> stride = layer.getIntTupleValue(AllPredefinedLayers.STRIDE_NAME).get();
ArchTypeSymbol inputType = layer.getInputTypes().get(0);
ArchTypeSymbol outputType = layer.getOutputTypes().get(0);
int heightWithPad = kernel.get(0) + stride.get(0)*(outputType.getHeight() - 1);
int widthWithPad = kernel.get(1) + stride.get(1)*(outputType.getWidth() - 1);
int heightPad = Math.max(0, heightWithPad - inputType.getHeight());
int widthPad = Math.max(0, widthWithPad - inputType.getWidth());
int topPad = (int)Math.ceil(heightPad / 2.0);
int bottomPad = (int)Math.floor(heightPad / 2.0);
int leftPad = (int)Math.ceil(widthPad / 2.0);
int rightPad = (int)Math.floor(widthPad / 2.0);
if (topPad == 0 && bottomPad == 0 && leftPad == 0 && rightPad == 0){
return null;
}
return Arrays.asList(0,0,0,0,topPad,bottomPad,leftPad,rightPad);
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnnarch.generator;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.monticar.cnnarch._cocos.CNNArchCocos;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchCompilationUnitSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchLanguage;
import de.monticore.symboltable.GlobalScope;
import de.monticore.symboltable.Scope;
import de.se_rwth.commons.logging.Log;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
public class CNNArchGenerator {
private String generationTargetPath;
public CNNArchGenerator() {
setGenerationTargetPath("./target/generated-sources-cnnarch/");
}
public String getGenerationTargetPath() {
if (generationTargetPath.charAt(generationTargetPath.length() - 1) != '/') {
this.generationTargetPath = generationTargetPath + "/";
}
return generationTargetPath;
}
public void setGenerationTargetPath(String generationTargetPath) {
this.generationTargetPath = generationTargetPath;
}
public void generate(Path modelsDirPath, String rootModelName){
final ModelPath mp = new ModelPath(modelsDirPath);
GlobalScope scope = new GlobalScope(mp, new CNNArchLanguage());
generate(scope, rootModelName);
}
public void generate(Scope scope, String rootModelName){
Optional<CNNArchCompilationUnitSymbol> compilationUnit = scope.resolve(rootModelName, CNNArchCompilationUnitSymbol.KIND);
if (!compilationUnit.isPresent()){
Log.error("could not resolve architecture " + rootModelName);
System.exit(1);
}
CNNArchCocos.checkAll(compilationUnit.get());
try{
generateFiles(compilationUnit.get().getArchitecture());
}
catch (IOException e){
Log.error(e.toString());
}
}
//check cocos with CNNArchCocos.checkAll(architecture) before calling this method.
public Map<String, String> generateStrings(ArchitectureSymbol architecture){
Map<String, String> fileContentMap = new HashMap<>();
CNNArchTemplateController archTc = new CNNArchTemplateController(architecture);
Map.Entry<String, String> temp;
temp = archTc.process("CNNPredictor", Target.CPP);
fileContentMap.put(temp.getKey(), temp.getValue());
temp = archTc.process("CNNCreator", Target.PYTHON);
fileContentMap.put(temp.getKey(), temp.getValue());
temp = archTc.process("execute", Target.CPP);
fileContentMap.put(temp.getKey().replace(".h", ""), temp.getValue());
temp = archTc.process("CNNBufferFile", Target.CPP);
fileContentMap.put("CNNBufferFile.h", temp.getValue());
checkValidGeneration(architecture);
return fileContentMap;
}
private void checkValidGeneration(ArchitectureSymbol architecture){
if (architecture.getInputs().size() > 1){
Log.warn("This cnn architecture has multiple inputs, " +
"which is currently not supported by the generator. " +
"The generated code will not work correctly."
, architecture.getSourcePosition());
}
if (architecture.getOutputs().size() > 1){
Log.warn("This cnn architecture has multiple outputs, " +
"which is currently not supported by the generator. " +
"The generated code will not work correctly."
, architecture.getSourcePosition());
}
if (architecture.getOutputs().get(0).getDefinition().getType().getWidth() != 1 ||
architecture.getOutputs().get(0).getDefinition().getType().getHeight() != 1){
Log.error("This cnn architecture has a multi-dimensional output, " +
"which is currently not supported by the generator."
, architecture.getSourcePosition());
}
}
//check cocos with CNNArchCocos.checkAll(architecture) before calling this method.
public void generateFiles(ArchitectureSymbol architecture) throws IOException{
CNNArchTemplateController archTc = new CNNArchTemplateController(architecture);
Map<String, String> fileContentMap = generateStrings(architecture);
for (String fileName : fileContentMap.keySet()){
File f = new File(getGenerationTargetPath() + fileName);
Log.info(f.getName(), "FileCreation:");
if (!f.exists()) {
f.getParentFile().mkdirs();
if (!f.createNewFile()) {
Log.error("File could not be created");
}
}
FileWriter writer = new FileWriter(f);
writer.write(fileContentMap.get(fileName));
writer.close();
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnnarch.generator;
import org.apache.commons.cli.*;
import java.nio.file.Path;
import java.nio.file.Paths;
public class CNNArchGeneratorCli {
public static final Option OPTION_MODELS_PATH = Option.builder("m")
.longOpt("models-dir")
.desc("full path to the directory with the CNNArch model")
.hasArg(true)
.required(true)
.build();
public static final Option OPTION_ROOT_MODEL = Option.builder("r")
.longOpt("root-model")
.desc("name of the architecture")
.hasArg(true)
.required(true)
.build();
public static final Option OPTION_OUTPUT_PATH = Option.builder("o")
.longOpt("output-dir")
.desc("full path to output directory for tests")
.hasArg(true)
.required(false)
.build();
private CNNArchGeneratorCli() {
}
public static void main(String[] args) {
Options options = getOptions();
CommandLineParser parser = new DefaultParser();
CommandLine cliArgs = parseArgs(options, parser, args);
if (cliArgs != null) {
runGenerator(cliArgs);
}
}
private static Options getOptions() {
Options options = new Options();
options.addOption(OPTION_MODELS_PATH);
options.addOption(OPTION_ROOT_MODEL);
options.addOption(OPTION_OUTPUT_PATH);
return options;
}
private static CommandLine parseArgs(Options options, CommandLineParser parser, String[] args) {
CommandLine cliArgs;
try {
cliArgs = parser.parse(options, args);
} catch (ParseException e) {
System.err.println("argument parsing exception: " + e.getMessage());
System.exit(1);
return null;
}
return cliArgs;
}
private static void runGenerator(CommandLine cliArgs) {
Path modelsDirPath = Paths.get(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()));
String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
String outputPath = cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt());
CNNArchGenerator generator = new CNNArchGenerator();
if (outputPath != null){
generator.setGenerationTargetPath(outputPath);
}
generator.generate(modelsDirPath, rootModelName);
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnnarch.generator;
import de.monticore.lang.monticar.cnnarch._symboltable.*;
import de.monticore.lang.monticar.cnnarch.predefined.Convolution;
import de.monticore.lang.monticar.cnnarch.predefined.FullyConnected;
import de.monticore.lang.monticar.cnnarch.predefined.Pooling;
import java.util.*;
public class LayerNameCreator {
private Map<ArchitectureElementSymbol, String> elementToName = new HashMap<>();
private Map<String, ArchitectureElementSymbol> nameToElement = new HashMap<>();
public LayerNameCreator(ArchitectureSymbol architecture) {
name(architecture.getBody(), 1, new ArrayList<>());
}
public ArchitectureElementSymbol getArchitectureElement(String name){
return nameToElement.get(name);
}
public String getName(ArchitectureElementSymbol architectureElement){
return elementToName.get(architectureElement);
}
protected int name(ArchitectureElementSymbol architectureElement, int stage, List<Integer> streamIndices){
if (architectureElement instanceof CompositeElementSymbol){
return nameComposite((CompositeElementSymbol) architectureElement, stage, streamIndices);
}
else{
if (architectureElement.isAtomic()){
if (architectureElement.getMaxSerialLength().get() > 0){
return add(architectureElement, stage, streamIndices);
}
else {
return stage;
}
}
else {
ArchitectureElementSymbol resolvedElement = architectureElement.getResolvedThis().get();
return name(resolvedElement, stage, streamIndices);
}
}
}
protected int nameComposite(CompositeElementSymbol compositeElement, int stage, List<Integer> streamIndices){
if (compositeElement.isParallel()){
int startStage = stage + 1;
streamIndices.add(1);
int lastIndex = streamIndices.size() - 1;
List<Integer> endStages = new ArrayList<>();
for (ArchitectureElementSymbol subElement : compositeElement.getElements()){
endStages.add(name(subElement, startStage, streamIndices));
streamIndices.set(lastIndex, streamIndices.get(lastIndex) + 1);
}
streamIndices.remove(lastIndex);
return Collections.max(endStages) + 1;
}
else {
int endStage = stage;
for (ArchitectureElementSymbol subElement : compositeElement.getElements()){
endStage = name(subElement, endStage, streamIndices);
}
return endStage;
}
}
protected int add(ArchitectureElementSymbol architectureElement, int stage, List<Integer> streamIndices){
int endStage = stage;
if (!elementToName.containsKey(architectureElement)) {
String name = createName(architectureElement, endStage, streamIndices);
while (nameToElement.containsKey(name)) {
endStage++;
name = createName(architectureElement, endStage, streamIndices);
}
elementToName.put(architectureElement, name);
nameToElement.put(name, architectureElement);
}
return endStage;
}
protected String createName(ArchitectureElementSymbol architectureElement, int stage, List<Integer> streamIndices){
if (architectureElement instanceof IOSymbol){
String name = createBaseName(architectureElement);
IOSymbol ioElement = (IOSymbol) architectureElement;
if (ioElement.getArrayAccess().isPresent()){
int arrayAccess = ioElement.getArrayAccess().get().getIntValue().get();
name = name + "_" + arrayAccess + "_";
}
return name;
}
else {
return createBaseName(architectureElement) + stage + createStreamPostfix(streamIndices) + "_";
}
}
protected String createBaseName(ArchitectureElementSymbol architectureElement){
if (architectureElement instanceof LayerSymbol) {
LayerDeclarationSymbol layerDeclaration = ((LayerSymbol) architectureElement).getDeclaration();
if (layerDeclaration instanceof Convolution) {
return "conv";
} else if (layerDeclaration instanceof FullyConnected) {
return "fc";
} else if (layerDeclaration instanceof Pooling) {
return "pool";
} else {
return layerDeclaration.getName().toLowerCase();
}
}
else if (architectureElement instanceof CompositeElementSymbol){
return "group";
}
else {
return architectureElement.getName();
}
}
protected String createStreamPostfix(List<Integer> streamIndices){
StringBuilder stringBuilder = new StringBuilder();
for (int streamIndex : streamIndices){
stringBuilder.append("_");
stringBuilder.append(streamIndex);
}
return stringBuilder.toString();
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,