Moved generator from EmbeddedMontiArcDL to this repo.

parent a9fc8933
target
nppBackup
.project
.settings
.classpath
.idea
.git
*.iml
script:
- git checkout ${TRAVIS_BRANCH}
- mvn clean install cobertura:cobertura org.eluder.coveralls:coveralls-maven-plugin:report --settings "settings.xml"
after_success:
- if [ "${TRAVIS_BRANCH}" == "master" ]; then mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B deploy --debug --settings "./settings.xml"; fi
This diff is collapsed.
******************************************************************************
MontiCAR Modeling Family, www.se-rwth.de
Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
All rights reserved.
This project is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this project. If not, see <http://www.gnu.org/licenses/>.
*******************************************************************************
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.generator;
import de.monticore.ModelingLanguageFamily;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.embeddedmontiarc.LogConfig;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ConstantPortSymbol;
import de.monticore.lang.monticar.emadl._symboltable.EMADLLanguage;
import de.monticore.lang.monticar.enumlang._symboltable.EnumLangLanguage;
import de.monticore.lang.monticar.generator.cpp.converter.MathConverter;
import de.monticore.lang.monticar.generator.optimization.ThreadingOptimizer;
import de.monticore.lang.monticar.generator.order.nfp.TagBreakpointsTagSchema.TagBreakpointsTagSchema;
import de.monticore.lang.monticar.generator.order.nfp.TagDelayTagSchema.TagDelayTagSchema;
import de.monticore.lang.monticar.generator.order.nfp.TagExecutionOrderTagSchema.TagExecutionOrderTagSchema;
import de.monticore.lang.monticar.generator.order.nfp.TagInitTagSchema.TagInitTagSchema;
import de.monticore.lang.monticar.generator.order.nfp.TagMinMaxTagSchema.TagMinMaxTagSchema;
import de.monticore.lang.monticar.generator.order.nfp.TagTableTagSchema.TagTableTagSchema;
import de.monticore.lang.monticar.generator.order.nfp.TagThresholdTagSchema.TagThresholdTagSchema;
import de.monticore.lang.monticar.streamunits._symboltable.StreamUnitsLanguage;
import de.monticore.lang.monticar.struct._symboltable.StructLanguage;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.monticore.symboltable.GlobalScope;
import de.monticore.symboltable.Scope;
import java.nio.file.Paths;
import java.util.Arrays;
public class AbstractSymtab {
public static TaggingResolver createSymTabAndTaggingResolver(String... modelPath) {
Scope scope = createSymTab(modelPath);
TaggingResolver tagging = new TaggingResolver(scope, Arrays.asList(modelPath));
TagMinMaxTagSchema.registerTagTypes(tagging);
TagTableTagSchema.registerTagTypes(tagging);
TagBreakpointsTagSchema.registerTagTypes(tagging);
TagExecutionOrderTagSchema.registerTagTypes(tagging);
TagInitTagSchema.registerTagTypes(tagging);
TagThresholdTagSchema.registerTagTypes(tagging);
TagDelayTagSchema.registerTagTypes(tagging);
return tagging;
}
public static Scope createSymTab(String... modelPath) {
ConstantPortSymbol.resetLastID();
MathConverter.resetIDs();
ThreadingOptimizer.resetID();
ModelingLanguageFamily fam = new ModelingLanguageFamily();
EMADLLanguage montiArcLanguage = new EMADLLanguage();
fam.addModelingLanguage(montiArcLanguage);
fam.addModelingLanguage(new StreamUnitsLanguage());
fam.addModelingLanguage(new StructLanguage());
fam.addModelingLanguage(new EnumLangLanguage());
final ModelPath mp = new ModelPath();
for (String m : modelPath) {
mp.addEntry(Paths.get(m));
}
LogConfig.init();//TODO comment for debug output
GlobalScope scope = new GlobalScope(mp, fam);
de.monticore.lang.monticar.Utils.addBuiltInTypes(scope);
return scope;
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.generator;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.apache.commons.cli.*;
import java.io.IOException;
public class EMADLGeneratorCli {
public static final Option OPTION_MODELS_PATH = Option.builder("m")
.longOpt("models-dir")
.desc("full path to directory with EMADL models e.g. C:\\Users\\vpupkin\\proj\\MyAwesomeAutopilot\\src\\main\\emam")
.hasArg(true)
.required(true)
.build();
public static final Option OPTION_ROOT_MODEL = Option.builder("r")
.longOpt("root-model")
.desc("fully qualified name of the root model e.g. de.rwth.vpupkin.modeling.mySuperAwesomeAutopilotComponent")
.hasArg(true)
.required(true)
.build();
public static final Option OPTION_OUTPUT_PATH = Option.builder("o")
.longOpt("output-dir")
.desc("full path to output directory for tests e.g. C:\\Users\\vpupkin\\proj\\MyAwesomeAutopilot\\target\\gen-cpp")
.hasArg(true)
.required(false)
.build();
private EMADLGeneratorCli() {
}
public static void main(String[] args) {
Options options = getOptions();
CommandLineParser parser = new DefaultParser();
CommandLine cliArgs = parseArgs(options, parser, args);
if (cliArgs != null) {
runGenerator(cliArgs);
}
}
private static Options getOptions() {
Options options = new Options();
options.addOption(OPTION_MODELS_PATH);
options.addOption(OPTION_ROOT_MODEL);
options.addOption(OPTION_OUTPUT_PATH);
return options;
}
private static CommandLine parseArgs(Options options, CommandLineParser parser, String[] args) {
CommandLine cliArgs;
try {
cliArgs = parser.parse(options, args);
} catch (ParseException e) {
System.err.println("argument parsing exception: " + e.getMessage());
System.exit(1);
return null;
}
return cliArgs;
}
private static void runGenerator(CommandLine cliArgs) {
String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
String outputPath = cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt());
EMADLGenerator generator = new EMADLGenerator();
if (outputPath != null){
generator.setGenerationTargetPath(outputPath);
}
try{
generator.generate(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()), rootModelName);
}
catch (IOException e){
Log.error("io error during generation", e);
System.exit(1);
}
catch (TemplateException e){
Log.error("template error during generation", e);
System.exit(1);
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl.generator;
import freemarker.template.Configuration;
import freemarker.template.TemplateExceptionHandler;
public class TemplateConfiguration {
private static TemplateConfiguration instance;
private Configuration configuration;
private TemplateConfiguration() {
configuration = new Configuration(Configuration.VERSION_2_3_23);
configuration.setClassForTemplateLoading(TemplateConfiguration.class, "/templates/");
configuration.setDefaultEncoding("UTF-8");
configuration.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
}
public Configuration getConfiguration() {
return configuration;
}
public static Configuration get(){
if (instance == null){
instance = new TemplateConfiguration();
}
return instance.getConfiguration();
}
}
import logging
import mxnet as mx
<#list instances as instance>
import CNNCreator_${instance.fullName?replace(".", "_")}
</#list>
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
logger.addHandler(handler)
<#list instances as instance>
${instance.fullName?replace(".", "_")} = CNNCreator_${instance.fullName?replace(".", "_")}.CNNCreator_${instance.fullName?replace(".", "_")}()
${instance.fullName?replace(".", "_")}.train(
<#if (trainParams[instance_index])??>
${trainParams[instance_index]}
</#if>
)
</#list>
\ No newline at end of file
#ifndef CNNTRANSLATOR_H
#define CNNTRANSLATOR_H
#include <armadillo>
#include <cassert>
using namespace std;
using namespace arma;
class CNNTranslator{
public:
template<typename T> static void addColToSTDVector(const Col<T> &source, vector<float> &data){
for(size_t i = 0; i < source.n_elem; i++){
data.push_back((float) source(i));
}
}
template<typename T> static void addRowToSTDVector(const subview_row<T> &source, vector<float> &data){
for(size_t i = 0; i < source.n_elem; i++){
data.push_back((float) source(i));
}
}
template<typename T> static void addRowToSTDVector(const Row<T> &source, vector<float> &data){
for(size_t i = 0; i < source.n_elem; i++){
data.push_back((float) source(i));
}
}
template<typename T> static void addMatToSTDVector(const Mat<T> &source, vector<float> &data){
for(size_t i = 0; i < source.n_rows; i++){
addRowToSTDVector(source.row(i), data);
}
}
template<typename T> static vector<float> translate(const Col<T> &source){
size_t size = source.n_elem;
vector<float> data;
data.reserve(size);
addColToSTDVector(source, data);
return data;
}
template<typename T> static vector<float> translate(const Row<T> &source){
size_t size = source.n_elem;
vector<float> data;
data.reserve(size);
addRowToSTDVector(source, data);
return data;
}
template<typename T> static vector<float> translate(const Mat<T> &source){
size_t size = source.n_elem;
vector<float> data;
data.reserve(size);
addMatToSTDVector(source, data);
return data;
}
template<typename T> static vector<float> translate(const Cube<T> &source){
size_t size = source.n_elem;
vector<float> data;
data.reserve(size);
for(size_t i = 0; i < source.n_slices; i++){
addMatToSTDVector(source.slice(i), data);
}
return data;
}
static vec translateToCol(const vector<float> &source, const vector<size_t> &shape){
assert(shape.size() == 1);
vec column(shape[0]);
for(size_t i = 0; i < source.size(); i++){
column(i) = (double) source[i];
}
return column;
}
static mat translateToMat(const vector<float> &source, const vector<size_t> &shape){
assert(shape.size() == 2);
mat matrix(shape[1], shape[0]); //create transposed version of the matrix
int startPos = 0;
int endPos = matrix.n_rows;
const vector<size_t> columnShape = {matrix.n_rows};
for(size_t i = 0; i < matrix.n_cols; i++){
vector<float> colSource(&source[startPos], &source[endPos]);
matrix.col(i) = translateToCol(colSource, columnShape);
startPos = endPos;
endPos += matrix.n_rows;
}
return matrix.t();
}
static cube translateToCube(const vector<float> &source, const vector<size_t> &shape){
assert(shape.size() == 3);
cube cubeMatrix(shape[1], shape[2], shape[0]);
const int matrixSize = shape[1] * shape[2];
const vector<size_t> matrixShape = {shape[1], shape[2]};
int startPos = 0;
int endPos = matrixSize;
for(size_t i = 0; i < cubeMatrix.n_slices; i++){
vector<float> matrixSource(&source[startPos], &source[endPos]);
cubeMatrix.slice(i) = translateToMat(matrixSource, matrixShape);
startPos = endPos;
endPos += matrixSize;
}
return cubeMatrix;
}
template<typename T> static vector<size_t> getShape(const Col<T> &source){
return {source.n_elem};
}
template<typename T> static vector<size_t> getShape(const Row<T> &source){
return {source.n_elem};
}
template<typename T> static vector<size_t> getShape(const Mat<T> &source){
return {source.n_rows, source.n_cols};
}
template<typename T> static vector<size_t> getShape(const Cube<T> &source){
return {source.n_slices, source.n_rows, source.n_cols};
}
};
#endif
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.AbstractSymtab;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import org.junit.Assert;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.assertTrue;
public class AbstractSymtabTest {
protected static TaggingResolver createSymTab(String... modelPath) {
return AbstractSymtab.createSymTabAndTaggingResolver(modelPath);
}
public static void checkFilesAreEqual(Path generationPath, Path resultsPath, List<String> fileNames) {
for (String fileName : fileNames){
File genFile = new File(generationPath.toString() + "/" + fileName);
File fileTarget = new File(resultsPath.toString() + "/" + fileName);
assertTrue(areBothFilesEqual(genFile, fileTarget));
}
}
public static boolean areBothFilesEqual(File file1, File file2) {
if (!file1.exists()) {
Assert.fail("file does not exist: " + file1.getAbsolutePath());
return false;
}
if (!file2.exists()) {
Assert.fail("file does not exist: " + file2.getAbsolutePath());
return false;
}
List<String> lines1;
List<String> lines2;
try {
lines1 = Files.readAllLines(file1.toPath());
lines2 = Files.readAllLines(file2.toPath());
} catch (IOException e) {
e.printStackTrace();
Assert.fail("IO error: " + e.getMessage());
return false;
}
lines1 = discardEmptyLines(lines1);
lines2 = discardEmptyLines(lines2);
if (lines1.size() != lines2.size()) {
Assert.fail(
"files have different number of lines: "
+ file1.getAbsolutePath()
+ " has " + lines1
+ " lines and " + file2.getAbsolutePath() + " has " + lines2 + " lines"
);
return false;
}
int len = lines1.size();
for (int i = 0; i < len; i++) {
String l1 = lines1.get(i);
String l2 = lines2.get(i);
Assert.assertEquals("files differ in " + i + " line: "
+ file1.getAbsolutePath()
+ " has " + l1
+ " and " + file2.getAbsolutePath() + " has " + l2,
l1,
l2
);
}
return true;
}
private static List<String> discardEmptyLines(List<String> lines) {
return lines.stream()
.map(String::trim)
.filter(l -> !l.isEmpty())
.collect(Collectors.toList());
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Arrays;
import static junit.framework.TestCase.assertTrue;
public class GenerationTest extends AbstractSymtabTest {
@Before
public void setUp() {
// ensure an empty log
Log.getFindings().clear();
Log.enableFailQuick(false);
}
@Test
public void testCifar10Generation() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code"),
Arrays.asList(
"cifar10_cifar10Classifier.h",
"CNNCreator_cifar10_cifar10Classifier_net.py",
"CNNBufferFile.h",
"CNNPredictor_cifar10_cifar10Classifier_net.h",
"cifar10_cifar10Classifier_net.h",
"CNNTranslator.h",
"cifar10_cifar10Classifier_calculateClass.h",
"CNNTrainer_cifar10_Cifar10Classifier.py"));
}
@Test
public void testSimulatorGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Test
public void testAddGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Add"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Test
public void testAlexnetGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Test
public void testResNeXtGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Test
public void testThreeInputGeneration() throws IOException, TemplateException {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().size() == 1);
}
@Test
public void testMultipleOutputsGeneration() throws IOException, TemplateException {
Log.getFindings().clear();<